diff options
| -rw-r--r-- | .appveyor.yml | 6 | ||||
| -rw-r--r-- | .travis.yml | 10 | ||||
| -rw-r--r-- | CONTRIBUTORS | 52 | ||||
| -rwxr-xr-x | release/rtool.py | 288 |
4 files changed, 168 insertions, 188 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index dae12978..7d5d7b1b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -31,12 +31,12 @@ deploy_script: ps: | if( ($Env:TOXENV -match "py35") -and - (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) + (($Env:APPVEYOR_REPO_BRANCH -match "builds") -or ($Env:APPVEYOR_REPO_TAG -match "true")) ) { pip install -U virtualenv .\dev.ps1 - cmd /c "python .\release\rtool.py bdist 2>&1" - python .\release\rtool.py upload-snapshot --bdist + cmd /c "python -u .\release\rtool.py bdist 2>&1" + python -u .\release\rtool.py upload-snapshot --bdist --wheel } cache: diff --git a/.travis.yml b/.travis.yml index e9566ebe..09474c05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,9 @@ matrix: - os: osx osx_image: xcode7.3 language: generic - env: TOXENV=py35 + env: TOXENV=py35 BDIST=1 - python: 3.5 - env: TOXENV=py35 + env: TOXENV=py35 BDIST=1 - python: 3.5 env: TOXENV=py35 NO_ALPN=1 - python: 2.7 @@ -57,14 +57,14 @@ script: set -o pipefail; python -m tox -- --cov netlib --cov mitmproxy --cov pat after_success: - | - if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "builds" || -n $TRAVIS_TAG) ]] then git fetch --unshallow ./dev.sh 3.5 source venv3.5/bin/activate pip install -e ./release - python ./release/rtool.py bdist - python ./release/rtool.py upload-snapshot --bdist --wheel + python -u ./release/rtool.py bdist + python -u ./release/rtool.py upload-snapshot --bdist fi notifications: diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 9609a421..a08e2eab 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,10 +1,13 @@ - 1813 Aldo Cortesi - 1228 Maximilian Hils - 282 Thomas Kriechbaumer + 2118 Aldo Cortesi + 1666 Maximilian Hils + 450 Thomas Kriechbaumer + 210 Shadab Zafar + 94 Jason 83 Marcelo Glezer + 36 Clemens 28 Jim Shaver 18 Henrik Nordstrom - 17 Shadab Zafar + 16 Matthew Shao 14 David Weinstein 14 Pedro Worcel 13 Thomas Roth @@ -14,39 +17,47 @@ 10 András Veres-Szentkirályi 10 Chris Czub 10 Sandor Nemes + 10 Zohar Lorberbaum 9 Kyle Morton 9 Legend Tang - 9 Matthew Shao 9 Rouli + 9 ikoz 8 Chandler Abraham 8 Jason A. Novak 7 Alexis Hildebrandt 7 Brad Peabody 7 Matthias Urlichs + 7 dufferzafar + 6 Felix Yan 5 Choongwoo Han 5 Sam Cleveland 5 Tomaz Muraus + 5 Will Coster 5 elitest 5 iroiro123 4 Bryan Bishop + 4 Clemens Brunner 4 Marc Liyanage 4 Michael J. Bazzinotti 4 Valtteri Virtanen 4 Wade 524 4 Youhei Sakurai 4 root + 4 yonder 3 Benjamin Lee 3 Chris Neasbitt 3 Eli Shvartsman - 3 Felix Yan 3 Guillem Anguera 3 Kyle Manna 3 MatthewShao 3 Ryan Welton 3 Zack B + 3 redfast00 + 3 requires.io 2 Anant 2 Bennett Blodinger 2 Colin Bendell + 2 Cory Benfield 2 Heikki Hannikainen 2 Israel Nir 2 Jaime Soriano Pastor @@ -59,34 +70,50 @@ 2 Paul 2 Rob Wills 2 Sean Coates + 2 Steven Van Acker 2 Terry Long 2 Wade Catron 2 alts 2 isra17 2 israel - 2 requires.io + 2 jpkrause + 2 lilydjwg + 2 strohu + 2 依云 + 1 Aditya 1 Andrey Plotnikov 1 Andy Smith + 1 Anthony Zhang + 1 BSalita 1 Ben Lerner 1 Bradley Baetz + 1 Brett Randall 1 Chris Hamant + 1 Christian Frichot 1 Dan Wilbraham 1 David Dworken 1 David Shaw + 1 Doug Freed 1 Doug Lethin + 1 Drake Caraker 1 Eric Entzel 1 Felix Wolfsteller 1 FreeArtMan 1 Gabriel Kirkpatrick 1 Henrik Nordström + 1 Israel Blancas 1 Ivaylo Popov 1 JC 1 Jakub Nawalaniec 1 Jakub Wilk 1 James Billingham + 1 Jason Pepas 1 Jean Regisser + 1 Jonathan Jones 1 Jorge Villacorta 1 Kit Randel + 1 Kostya Esmukov + 1 Linmiao Xu 1 Lucas Cimon 1 M. Utku Altinkaya 1 Mathieu Mitchell @@ -98,27 +125,33 @@ 1 Nick Raptis 1 Nicolas Esteves 1 Oleksandr Sheremet + 1 Parth Ganatra 1 Pritam Baral 1 Rich Somerfield 1 Rory McCann 1 Rune Halvorsen 1 Ryo Onodera + 1 Sachin Kelkar 1 Sahn Lam 1 Seppo Yli-Olli 1 Sergey Chipiga 1 Stefan Wärting 1 Steve Phillips - 1 Steven Van Acker + 1 Steven Noble 1 Suyash + 1 Tai Dickerson 1 Tarashish Mishra 1 TearsDontFalls + 1 Thiago Arrais 1 Tim Becker 1 Timothy Elliott 1 Ulrich Petri 1 Vyacheslav Bakhmutov - 1 Will Coster + 1 Wes Turner + 1 Yoginski 1 Yuangxuan Wang 1 capt8bit + 1 cle1000 1 davidpshaw 1 deployable 1 gecko655 @@ -133,4 +166,3 @@ 1 sethp-jive 1 starenka 1 vzvu3k6k - 1 依云 diff --git a/release/rtool.py b/release/rtool.py index 4e43eaef..051a5188 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,29 +1,30 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import absolute_import, print_function, division -from os.path import join + import contextlib +import fnmatch import os +import platform +import runpy +import shlex import shutil import subprocess -import re -import shlex -import runpy -import zipfile +import sys import tarfile -import platform +import zipfile +from os.path import join, normpath, dirname, exists, basename + import click import pysftp -import fnmatch # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -import sys - if platform.system() == "Windows": VENV_BIN = "Scripts" else: VENV_BIN = "bin" +# ZipFile and tarfile have slightly different APIs. Fix that. if platform.system() == "Windows": def Archive(name): a = zipfile.ZipFile(name, "w") @@ -33,13 +34,13 @@ else: def Archive(name): return tarfile.open(name, "w:gz") -RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) -DIST_DIR = join(RELEASE_DIR, "dist") -ROOT_DIR = os.path.normpath(join(RELEASE_DIR, "..")) -RELEASE_SPEC_DIR = join(RELEASE_DIR, "specs") -VERSION_FILE = join(ROOT_DIR, "netlib/version.py") +ROOT_DIR = normpath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") PYINSTALLER_DIST = join(BUILD_DIR, "binaries") @@ -47,27 +48,35 @@ VENV_DIR = join(BUILD_DIR, "venv") VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") -project = { - "name": "mitmproxy", - "tools": ["pathod", "pathoc", "mitmproxy", "mitmdump", "mitmweb"], - "bdists": { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] - }, - "dir": ROOT_DIR, - "python_version": "py2.py3", +# Project Configuration +VERSION_FILE = join(ROOT_DIR, "netlib", "version.py") +PROJECT_NAME = "mitmproxy" +PYTHON_VERSION = "py2.py3" +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] } if platform.system() == "Windows": - project["tools"].remove("mitmproxy") - project["bdists"]["mitmproxy"].remove("mitmproxy") + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in BDISTS.values() + for tool in tools +] -def get_version(): +def get_version() -> str: return runpy.run_path(VERSION_FILE)["VERSION"] -def get_snapshot_version(): - last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit(b"-", 2) +def git(args: str) -> str: + with chdir(ROOT_DIR): + return subprocess.check_output(["git"] + shlex.split(args)).decode() + + +def get_snapshot_version() -> str: + last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) tag_dist = int(tag_dist) if tag_dist == 0: return get_version() @@ -76,11 +85,11 @@ def get_snapshot_version(): return "{version}dev{tag_dist:04}-0x{commit}".format( version=get_version(), # this should already be the next version tag_dist=tag_dist, - commit=commit.decode() + commit=commit ) -def archive_name(project): +def archive_name(bdist: str) -> str: platform_tag = { "Darwin": "osx", "Windows": "win32", @@ -91,18 +100,18 @@ def archive_name(project): else: ext = "tar.gz" return "{project}-{version}-{platform}.{ext}".format( - project=project, + project=bdist, version=get_version(), platform=platform_tag, ext=ext ) -def wheel_name(): +def wheel_name() -> str: return "{project}-{version}-{py_version}-none-any.whl".format( - project=project["name"], + project=PROJECT_NAME, version=get_version(), - py_version=project["python_version"] + py_version=PYTHON_VERSION ) @@ -119,18 +128,13 @@ def empty_pythonpath(): @contextlib.contextmanager -def chdir(path): +def chdir(path: str): old_dir = os.getcwd() os.chdir(path) yield os.chdir(old_dir) -def git(args): - with chdir(ROOT_DIR): - return subprocess.check_output(["git"] + shlex.split(args)) - - @click.group(chain=True) def cli(): """ @@ -147,95 +151,79 @@ def contributors(): with chdir(ROOT_DIR): print("Updating CONTRIBUTORS...") contributors_data = git("shortlog -n -s") - with open("CONTRIBUTORS", "w") as f: - f.write(contributors_data) + with open("CONTRIBUTORS", "wb") as f: + f.write(contributors_data.encode()) -@cli.command("set-version") -@click.argument('version') -def set_version(version): +@cli.command("wheel") +def make_wheel(): """ - Update version information - """ - print("Update versions...") - version = ", ".join(version.split(".")) - print("Update %s..." % VERSION_FILE) - with open(VERSION_FILE, "rb") as f: - content = f.read() - new_content = re.sub( - r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, - content - ) - with open(VERSION_FILE, "wb") as f: - f.write(new_content) - - -@cli.command("wheels") -def wheels(): - """ - Build wheels + Build wheel """ with empty_pythonpath(): - print("Building release...") - if os.path.exists(DIST_DIR): + if exists(DIST_DIR): shutil.rmtree(DIST_DIR) - print("Creating wheel for %s ..." % project["name"]) + print("Creating wheel...") subprocess.check_call( [ "python", "./setup.py", "-q", "bdist_wheel", "--dist-dir", DIST_DIR, "--universal" ], - cwd=project["dir"] + cwd=ROOT_DIR ) print("Creating virtualenv for test install...") - if os.path.exists(VENV_DIR): + if exists(VENV_DIR): shutil.rmtree(VENV_DIR) subprocess.check_call(["virtualenv", "-q", VENV_DIR]) with chdir(DIST_DIR): - print("Installing %s..." % project["name"]) + print("Install wheel into virtualenv...") # lxml... if platform.system() == "Windows" and sys.version_info[0] == 3: - subprocess.check_call([VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]) + subprocess.check_call( + [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"] + ) subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()]) - print("Running binaries...") - for tool in project["tools"]: + print("Running tools...") + for tool in TOOLS: tool = join(VENV_DIR, VENV_BIN, tool) print("> %s --version" % tool) - print(subprocess.check_output([tool, "--version"])) + print(subprocess.check_output([tool, "--version"]).decode()) print("Virtualenv available for further testing:") - print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate"))) + print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate"))) @cli.command("bdist") -@click.option("--use-existing-wheels/--no-use-existing-wheels", default=False) +@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False) @click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.1.1") +@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1") @click.pass_context -def bdist(ctx, use_existing_wheels, pyinstaller_version): +def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version): """ Build a binary distribution """ - if os.path.exists(PYINSTALLER_TEMP): + if exists(PYINSTALLER_TEMP): shutil.rmtree(PYINSTALLER_TEMP) - if os.path.exists(PYINSTALLER_DIST): + if exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) - if not use_existing_wheels: - ctx.invoke(wheels) + if not use_existing_wheel: + ctx.invoke(make_wheel) - print("Installing PyInstaller...") - subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version]) + print("Installing PyInstaller and setuptools...") + subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version]) + print(subprocess.check_output([VENV_PIP, "freeze"]).decode()) - for bdist_project, tools in project["bdists"].items(): - with Archive(join(DIST_DIR, archive_name(bdist_project))) as archive: + for bdist, tools in BDISTS.items(): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: for tool in tools: # This is PyInstaller, so it messes up paths. # We need to make sure that we are in the spec folder. - with chdir(RELEASE_SPEC_DIR): + with chdir(PYINSTALLER_SPEC): print("Building %s binary..." % tool) subprocess.check_call( [ @@ -255,10 +243,10 @@ def bdist(ctx, use_existing_wheels, pyinstaller_version): if platform.system() == "Windows": executable += ".exe" print("> %s --version" % executable) - subprocess.check_call([executable, "--version"]) + print(subprocess.check_output([executable, "--version"]).decode()) - archive.add(executable, os.path.basename(executable)) - print("Packed {}.".format(archive_name(bdist_project))) + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) @cli.command("upload-release") @@ -298,89 +286,49 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, username=user, private_key=private_key, private_key_pass=private_key_password) as sftp: - - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - for bdist in project["bdists"].keys(): + dir_name = "snapshots/v{}".format(get_version()) + sftp.makedirs(dir_name) + with sftp.cd(dir_name): + files = [] + if wheel: + files.append(wheel_name()) + if bdist: + for bdist in BDISTS.keys(): files.append(archive_name(bdist)) - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = f.replace(get_version(), get_snapshot_version()) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - # We hide the file during upload. - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) - - -@cli.command("wizard") -@click.option('--next-version', prompt=True) -@click.option('--username', prompt="PyPI Username") -@click.password_option(confirmation_prompt=False, prompt="PyPI Password") -@click.option('--repository', default="pypi") -@click.pass_context -def wizard(ctx, next_version, username, password, repository): - """ - Interactive Release Wizard - """ - is_dirty = git("status --porcelain") - if is_dirty: - raise RuntimeError("Repository is not clean.") - - # update contributors file - ctx.invoke(contributors) - - # Build test release - ctx.invoke(bdist) - - try: - click.confirm("Please test the release now. Is it ok?", abort=True) - except click.Abort: - # undo changes - git("checkout CONTRIBUTORS") - raise - - # Everything ok - let's ship it! - git("tag v{}".format(get_version())) - git("push --tags") - ctx.invoke( - upload_release, - username=username, password=password, repository=repository - ) - - click.confirm("Now please wait until CI has built binaries. Finished?") - - # version bump commit - ctx.invoke(set_version, version=next_version) - git("commit -a -m \"bump version\"") - git("push") + for f in files: + local_path = join(DIST_DIR, f) + remote_filename = f.replace(get_version(), get_snapshot_version()) + symlink_path = "../{}".format(f.replace(get_version(), "latest")) + + # Upload new version + print("Uploading {} as {}...".format(f, remote_filename)) + with click.progressbar(length=os.stat(local_path).st_size) as bar: + # We hide the file during upload + sftp.put( + local_path, + "." + remote_filename, + callback=lambda done, total: bar.update(done - bar.pos) + ) - click.echo("All done!") + # Delete old versions + old_version = f.replace(get_version(), "*") + for f_old in sftp.listdir(): + if fnmatch.fnmatch(f_old, old_version): + print("Removing {}...".format(f_old)) + sftp.remove(f_old) + + # Show new version + sftp.rename("." + remote_filename, remote_filename) + + # update symlink for the latest release + if sftp.lexists(symlink_path): + print("Removing {}...".format(symlink_path)) + sftp.remove(symlink_path) + if f != wheel_name(): + # "latest" isn't a proper wheel version, so this could not be installed. + # https://github.com/mitmproxy/mitmproxy/issues/1065 + sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) if __name__ == "__main__": |