From 937a7daf1ffbec28e06ecda9be28b17d776fc442 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 21 Mar 2020 15:14:46 -0400 Subject: Move wheel builer to github actions (#5147) --- .azure-pipelines/wheel-builder.yml | 79 ------------------------------ .github/workflows/wheel-builder.yml | 54 +++++++++++++++++++++ release.py | 97 +++++++++++++++++++++++++++++++++++-- 3 files changed, 147 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/wheel-builder.yml diff --git a/.azure-pipelines/wheel-builder.yml b/.azure-pipelines/wheel-builder.yml index 47668263..6fcb3bc1 100644 --- a/.azure-pipelines/wheel-builder.yml +++ b/.azure-pipelines/wheel-builder.yml @@ -150,82 +150,3 @@ jobs: inputs: pathToPublish: wheelhouse/ artifactName: cryptography-$(PLATFORM)-$(PYTHON_VERSION) - - - job: 'windows' - pool: - vmImage: 'windows-2019' - container: $[variables.containerImage] - strategy: - matrix: - Python27-x86: - containerImage: 'pyca/cryptography-runner-windows:py27-x86' - PYTHON_VERSION: '27' - OPENSSL_DIR: 'OpenSSL-Win32-2010' - WINDOWS_ARCH: 'x86' - Python27-x86-64: - containerImage: 'pyca/cryptography-runner-windows:py27-x86_64' - PYTHON_VERSION: '27' - OPENSSL_DIR: 'OpenSSL-Win64-2010' - WINDOWS_ARCH: 'x86_64' - Python35-x86: - containerImage: 'pyca/cryptography-runner-windows:py35-x86' - PYTHON_VERSION: '35' - OPENSSL_DIR: 'OpenSSL-Win32-2015' - WINDOWS_ARCH: 'x86' - Python35-x86-64: - containerImage: 'pyca/cryptography-runner-windows:py35-x86_64' - PYTHON_VERSION: '35' - OPENSSL_DIR: 'OpenSSL-Win64-2015' - WINDOWS_ARCH: 'x86_64' - Python36-x86: - containerImage: 'pyca/cryptography-runner-windows:py3-x86' - PYTHON_VERSION: '36' - OPENSSL_DIR: 'OpenSSL-Win32-2015' - WINDOWS_ARCH: 'x86' - Python36-x86-64: - containerImage: 'pyca/cryptography-runner-windows:py3-x86_64' - PYTHON_VERSION: '36' - OPENSSL_DIR: 'OpenSSL-Win64-2015' - WINDOWS_ARCH: 'x86_64' - Python37-x86: - containerImage: 'pyca/cryptography-runner-windows:py3-x86' - PYTHON_VERSION: '37' - OPENSSL_DIR: 'OpenSSL-Win32-2015' - WINDOWS_ARCH: 'x86' - Python37-x86-64: - containerImage: 'pyca/cryptography-runner-windows:py3-x86_64' - PYTHON_VERSION: '37' - OPENSSL_DIR: 'OpenSSL-Win64-2015' - WINDOWS_ARCH: 'x86_64' - Python38-x86: - containerImage: 'pyca/cryptography-runner-windows:py3-x86' - PYTHON_VERSION: '38' - OPENSSL_DIR: 'OpenSSL-Win32-2015' - WINDOWS_ARCH: 'x86' - Python38-x86-64: - containerImage: 'pyca/cryptography-runner-windows:py3-x86_64' - PYTHON_VERSION: '38' - OPENSSL_DIR: 'OpenSSL-Win64-2015' - WINDOWS_ARCH: 'x86_64' - steps: - - script: | - "C:/Python%PYTHON_VERSION%/Scripts/pip" install cffi six ipaddress "enum34; python_version < '3'" - displayName: Install wheel and our Python dependencies - - script: | - set INCLUDE=C:/%OPENSSL_DIR%/include;%INCLUDE% - set LIB=C:/%OPENSSL_DIR%/lib;%LIB% - C:/Python%PYTHON_VERSION%/Scripts/pip wheel cryptography==%BUILD_VERSION% --wheel-dir=wheelhouse --no-binary cryptography - displayName: Build the wheel - - script: '"C:/Python%PYTHON_VERSION%/Scripts/pip" install -f wheelhouse cryptography --no-index' - displayName: Test installing the wheel - - script: | - "C:/Python%PYTHON_VERSION%/python" -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" - displayName: Print the OpenSSL we built and linked against - - script: mkdir cryptography-wheelhouse - displayName: Create a directory for placing the final wheel in - - script: move wheelhouse\cryptography*.whl cryptography-wheelhouse\ - displayName: Move the cryptography wheel into the final wheel house - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: cryptography-wheelhouse/ - artifactName: cryptography-windows-$(WINDOWS_ARCH)-python$(PYTHON_VERSION) diff --git a/.github/workflows/wheel-builder.yml b/.github/workflows/wheel-builder.yml new file mode 100644 index 00000000..888546cf --- /dev/null +++ b/.github/workflows/wheel-builder.yml @@ -0,0 +1,54 @@ +on: + repository_dispatch: + types: [wheel-builder] + +jobs: + windows: + strategy: + matrix: + WINDOWS: + - {ARCH: 'x86', WINDOWS: 'win32'} + - {ARCH: 'x64', WINDOWS: 'win64'} + PYTHON: + - {VERSION: "2.7", TOXENV: "py27", MSVC_VERSION: "2010"} + - {VERSION: "3.5", TOXENV: "py35", MSVC_VERSION: "2019"} + - {VERSION: "3.6", TOXENV: "py36", MSVC_VERSION: "2019"} + - {VERSION: "3.7", TOXENV: "py37", MSVC_VERSION: "2019"} + - {VERSION: "3.8", TOXENV: "py38", MSVC_VERSION: "2019"} + name: "Python ${{ matrix.PYTHON.VERSION }} on ${{ matrix.WINDOWS.WINDOWS }}" + steps: + - uses: actions/checkout@master + - name: Setup python + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.PYTHON.VERSION }} + architecture: ${{ matrix.WINDOWS.ARCH }} + - name: Install MSVC for Python 2.7 + run: | + Invoke-WebRequest -Uri https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi -OutFile VCForPython27.msi + Start-Process msiexec -Wait -ArgumentList @('/i', 'VCForPython27.msi', '/qn', 'ALLUSERS=1') + Remove-Item VCForPython27.msi -Force + shell: powershell + if: matrix.PYTHON.VERSION == '2.7' + - run: pip install requests + - name: Download OpenSSL + run: | + python .github/workflows/download_openssl.py openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }} + echo "::set-env name=INCLUDE::C:/openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }}/include;%INCLUDE%" + echo "::set-env name=LIB::C:/openssl-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.MSVC_VERSION }}/lib;%LIB%" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - run: pip install cffi six ipaddress "enum34; python_version < '3'" + - run: pip wheel cryptography==${{ github.event.client_payload.BUILD_VERSION }} --wheel-dir=wheelhouse --no-binary cryptography + - run: pip install -f wheelhouse --no-index cryptography + - name: Print the OpenSSL we built and linked against + run: | + python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" + + - run: mkdir cryptography-wheelhouse + - run: move wheelhouse\cryptography*.whl cryptography-wheelhouse\ + - uses: actions/upload-artifact@v1 + with: + name: "cryptography-${{ github.event.client_payload.BUILD_VERSION }}-${{ matrix.WINDOWS.WINDOWS }}-${{ matrix.PYTHON.VERSION }}" + path: cryptography-wheelhouse\ diff --git a/release.py b/release.py index c03d22d5..a2219e6e 100644 --- a/release.py +++ b/release.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function import getpass import glob +import io import json import os import subprocess @@ -20,6 +21,8 @@ import click from msrest.authentication import BasicAuthentication +import requests + def run(*args, **kwargs): print("[running] {0}".format(list(args))) @@ -61,8 +64,7 @@ def download_artifacts_azure(build_client, build_id): return paths -def build_wheels_azure(version): - token = getpass.getpass("Azure personal access token: ") +def build_wheels_azure(token, version): credentials = BasicAuthentication("", token) connection = Connection( base_url="https://dev.azure.com/pyca", creds=credentials @@ -82,12 +84,96 @@ def build_wheels_azure(version): return download_artifacts_azure(build_client, build.id) +def wait_for_build_complete_github_actions(session, token, run_url): + while True: + response = session.get(run_url, headers={ + "Content-Type": "application/json", + "Authorization": "token {}".format(token), + }) + response.raise_for_status() + if response.json()["conclusion"] is not None: + break + time.sleep(3) + + +def download_artifacts_github_actions(session, token, run_url): + response = session.get(run_url, headers={ + "Content-Type": "application/json", + "Authorization": "token {}".format(token), + }) + response.raise_for_status() + + response = session.get(response.json()["artifacts_url"], headers={ + "Content-Type": "application/json", + "Authorization": "token {}".format(token), + }) + response.raise_for_status() + paths = [] + for artifact in response.json()["artifacts"]: + response = session.get(artifact["archive_download_url"], headers={ + "Content-Type": "application/json", + "Authorization": "token {}".format(token), + }) + with zipfile.ZipFile(io.BytesIO(response.content)) as z: + for name in z.namelist(): + if not name.endswith(".whl"): + continue + p = z.open(name) + out_path = os.path.join( + os.path.dirname(__file__), + "dist", + os.path.basename(name), + ) + with open(out_path, "wb") as f: + f.write(p.read()) + paths.append(out_path) + return paths + + +def build_github_actions_wheels(token, version): + session = requests.Session() + + response = session.post( + "https://api.github.com/repos/pyca/cryptography/dispatches", + headers={ + "Content-Type": "application/json", + "Accept": "application/vnd.github.everest-preview+json", + "Authorization": "token {}".format(token), + }, + data=json.dumps({ + "event_type": "wheel-builder", + "client_payload": { + "BUILD_VERSION": version, + }, + }), + ) + response.raise_for_status() + + response = session.get( + ( + "https://api.github.com/repos/pyca/cryptography/actions/workflows/" + "wheel-builder.yml/runs?event=repository_dispatch" + ), + headers={ + "Content-Type": "application/json", + "Authorization": "token {}".format(token), + }, + ) + response.raise_for_status() + run_url = response.json()["workflow_runs"][0]["url"] + wait_for_build_complete_github_actions(session, token, run_url) + return download_artifacts_github_actions(session, token, run_url) + + @click.command() @click.argument("version") def release(version): """ ``version`` should be a string like '0.4' or '1.0'. """ + azure_token = getpass.getpass("Azure personal access token: ") + github_token = getpass.getpass("Github person access token: ") + run("git", "tag", "-s", version, "-m", "{0} release".format(version)) run("git", "push", "--tags") @@ -100,8 +186,11 @@ def release(version): ) run("twine", "upload", "-s", *packages) - azure_wheel_paths = build_wheels_azure(version) - run("twine", "upload", *azure_wheel_paths) + azure_wheel_paths = build_wheels_azure(azure_token, version) + github_actions_wheel_paths = build_github_actions_wheels( + github_token, version + ) + run("twine", "upload", *(azure_wheel_paths, github_actions_wheel_paths)) if __name__ == "__main__": -- cgit v1.2.3