diff options
| -rw-r--r-- | .appveyor.yml | 15 | ||||
| -rw-r--r-- | .travis.yml | 19 | ||||
| -rw-r--r-- | dev.ps1 | 2 | ||||
| -rw-r--r-- | mitmproxy/tools/__init__.py | 5 | ||||
| -rw-r--r-- | mitmproxy/utils/data.py | 4 | ||||
| -rw-r--r-- | mitmproxy/version.py | 3 | ||||
| -rw-r--r-- | release/hooks/hook-cryptography.py | 43 | ||||
| -rw-r--r-- | release/hooks/hook-mitmproxy.addons.onboardingapp.py | 2 | ||||
| -rw-r--r-- | release/hooks/hook-mitmproxy.tools.web.py | 3 | ||||
| -rwxr-xr-x | release/rtool.py | 126 | ||||
| -rw-r--r-- | release/setup.py | 2 | ||||
| -rw-r--r-- | release/specs/mitmproxy_main (renamed from release/specs/mitmproxy) | 0 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | setup.py | 3 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_script.py | 4 | ||||
| -rw-r--r-- | tox.ini | 27 |
16 files changed, 141 insertions, 118 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 746e0fd1..5421eb5a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -24,17 +24,20 @@ install: test_script: - ps: "tox -- --cov mitmproxy --cov pathod -v" + - ps: | + $Env:VERSION = $(python mitmproxy/version.py) + $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`"" + tox -e wheel -- https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl + tox -e rtool -- bdist deploy_script: + # we build binaries on every run, but we only upload them for master snapshots or tags. ps: | if( ($Env:TOXENV -match "py35") -and - (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) - ) { - pip install -U virtualenv - .\dev.ps1 - cmd /c "python -u .\release\rtool.py bdist 2>&1" - python -u .\release\rtool.py upload-snapshot --bdist --wheel + (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) + ) { + tox -e rtool -- upload-snapshot --bdist --wheel } cache: diff --git a/.travis.yml b/.travis.yml index 11e2832a..0df32899 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,18 +51,21 @@ install: fi - pip install tox -script: tox -- --cov mitmproxy --cov pathod -v +script: + - | + tox -- --cov mitmproxy --cov pathod -v + if [[ $BDIST == "1" ]] + then + git fetch --unshallow --tags + tox -e rtool -- bdist + fi after_success: + # we build binaries on every run, but we only upload them for master snapshots or tags. - | - if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then - git fetch --unshallow - ./dev.sh 3.5 - source venv3.5/bin/activate - pip install -e ./release - python -u ./release/rtool.py bdist - python -u ./release/rtool.py upload-snapshot --bdist + tox -e rtool -- upload-snapshot --bdist fi notifications: @@ -1,7 +1,7 @@ $ErrorActionPreference = "Stop" $VENV = ".\venv" -virtualenv $VENV --always-copy +python3 -m venv $VENV --copies & $VENV\Scripts\activate.ps1 python -m pip install --disable-pip-version-check -U pip diff --git a/mitmproxy/tools/__init__.py b/mitmproxy/tools/__init__.py index 5a8593df..e69de29b 100644 --- a/mitmproxy/tools/__init__.py +++ b/mitmproxy/tools/__init__.py @@ -1,5 +0,0 @@ -from mitmproxy.tools import web -from mitmproxy.tools import console -from mitmproxy.tools import dump - -__all__ = ["web", "console", "dump"] diff --git a/mitmproxy/utils/data.py b/mitmproxy/utils/data.py index 12298258..5a175fce 100644 --- a/mitmproxy/utils/data.py +++ b/mitmproxy/utils/data.py @@ -15,7 +15,7 @@ class Data: """ Change the data object to a path relative to the module. """ - dirname = os.path.join(self.dirname, subpath) + dirname = os.path.normpath(os.path.join(self.dirname, subpath)) ret = Data(self.name) ret.dirname = dirname return ret @@ -27,7 +27,7 @@ class Data: This function will raise ValueError if the path does not exist. """ - fullpath = os.path.join(self.dirname, path) + fullpath = os.path.normpath(os.path.join(self.dirname, path)) if not os.path.exists(fullpath): raise ValueError("dataPath: %s does not exist." % fullpath) return fullpath diff --git a/mitmproxy/version.py b/mitmproxy/version.py index cb670642..922c3218 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -2,3 +2,6 @@ IVERSION = (0, 19) VERSION = ".".join(str(i) for i in IVERSION) PATHOD = "pathod " + VERSION MITMPROXY = "mitmproxy " + VERSION + +if __name__ == "__main__": + print(VERSION) diff --git a/release/hooks/hook-cryptography.py b/release/hooks/hook-cryptography.py new file mode 100644 index 00000000..d53a438b --- /dev/null +++ b/release/hooks/hook-cryptography.py @@ -0,0 +1,43 @@ +# Taken from the latest pyinstaller master on 2016-11-27 (0729a2b). + +#----------------------------------------------------------------------------- +# Copyright (c) 2005-2016, PyInstaller Development Team. +# +# Distributed under the terms of the GNU General Public License with exception +# for distributing bootloader. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + + +""" +Hook for cryptography module from the Python Cryptography Authority. +""" + +import os.path +import glob + +from PyInstaller.compat import EXTENSION_SUFFIXES +from PyInstaller.utils.hooks import collect_submodules, get_module_file_attribute +from PyInstaller.utils.hooks import copy_metadata + +# get the package data so we can load the backends +datas = copy_metadata('cryptography') + +# Add the backends as hidden imports +hiddenimports = collect_submodules('cryptography.hazmat.backends') + +# Add the OpenSSL FFI binding modules as hidden imports +hiddenimports += collect_submodules('cryptography.hazmat.bindings.openssl') + ['_cffi_backend'] + + +# Include the cffi extensions as binaries in a subfolder named like the package. +# The cffi verifier expects to find them inside the package directory for +# the main module. We cannot use hiddenimports because that would add the modules +# outside the package. +binaries = [] +cryptography_dir = os.path.dirname(get_module_file_attribute('cryptography')) +for ext in EXTENSION_SUFFIXES: + ffimods = glob.glob(os.path.join(cryptography_dir, '*_cffi_*%s*' % ext)) + for f in ffimods: + binaries.append((f, 'cryptography'))
\ No newline at end of file diff --git a/release/hooks/hook-mitmproxy.addons.onboardingapp.py b/release/hooks/hook-mitmproxy.addons.onboardingapp.py index 1587f40a..2b2fe06b 100644 --- a/release/hooks/hook-mitmproxy.addons.onboardingapp.py +++ b/release/hooks/hook-mitmproxy.addons.onboardingapp.py @@ -1,3 +1,3 @@ from PyInstaller.utils.hooks import collect_data_files -datas = collect_data_files('mitmproxy.addons.onboardingapp')
\ No newline at end of file +datas = collect_data_files('mitmproxy.addons.onboardingapp') diff --git a/release/hooks/hook-mitmproxy.tools.web.py b/release/hooks/hook-mitmproxy.tools.web.py new file mode 100644 index 00000000..519c4c00 --- /dev/null +++ b/release/hooks/hook-mitmproxy.tools.web.py @@ -0,0 +1,3 @@ +from PyInstaller.utils.hooks import collect_data_files + +datas = collect_data_files('mitmproxy.tools.web') diff --git a/release/rtool.py b/release/rtool.py index 3bc3fdaa..f40e8dcb 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -8,10 +8,9 @@ import runpy import shlex import shutil import subprocess -import sys import tarfile import zipfile -from os.path import join, abspath, normpath, dirname, exists, basename +from os.path import join, abspath, dirname, exists, basename import click import pysftp @@ -20,8 +19,13 @@ import pysftp # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ if platform.system() == "Windows": VENV_BIN = "Scripts" + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] else: VENV_BIN = "bin" + PYINSTALLER_ARGS = [] # ZipFile and tarfile have slightly different APIs. Fix that. if platform.system() == "Windows": @@ -46,13 +50,9 @@ PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") PYINSTALLER_DIST = join(BUILD_DIR, "binaries") VENV_DIR = join(BUILD_DIR, "venv") -VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") -VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") # Project Configuration VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") -PROJECT_NAME = "mitmproxy" -PYTHON_VERSION = "py2.py3" BDISTS = { "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], "pathod": ["pathoc", "pathod"] @@ -62,20 +62,20 @@ if platform.system() == "Windows": TOOLS = [ tool - for tools in BDISTS.values() + for tools in sorted(BDISTS.values()) for tool in tools ] -def get_version() -> str: - return runpy.run_path(VERSION_FILE)["VERSION"] - - def git(args: str) -> str: with chdir(ROOT_DIR): return subprocess.check_output(["git"] + shlex.split(args)).decode() +def get_version() -> str: + return runpy.run_path(VERSION_FILE)["VERSION"] + + def get_snapshot_version() -> str: last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) tag_dist = int(tag_dist) @@ -109,26 +109,12 @@ def archive_name(bdist: str) -> str: def wheel_name() -> str: - return "{project}-{version}-{py_version}-none-any.whl".format( - project=PROJECT_NAME, + return "mitmproxy-{version}-py3-none-any.whl".format( version=get_version(), - py_version=PYTHON_VERSION ) @contextlib.contextmanager -def empty_pythonpath(): - """ - Make sure that the regular python installation is not on the python path, - which would give us access to modules installed outside of our virtualenv. - """ - pythonpath = os.environ.get("PYTHONPATH", "") - os.environ["PYTHONPATH"] = "" - yield - os.environ["PYTHONPATH"] = pythonpath - - -@contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() os.chdir(path) @@ -156,58 +142,8 @@ def contributors(): f.write(contributors_data.encode()) -@cli.command("wheel") -def make_wheel(): - """ - Build wheel - """ - with empty_pythonpath(): - if exists(DIST_DIR): - shutil.rmtree(DIST_DIR) - - print("Creating wheel...") - subprocess.check_call( - [ - "python3", "./setup.py", "-q", - "bdist_wheel", "--dist-dir", DIST_DIR, "--universal" - ], - cwd=ROOT_DIR - ) - - print("Creating virtualenv for test install...") - if exists(VENV_DIR): - shutil.rmtree(VENV_DIR) - subprocess.check_call(["python3", "-m", "virtualenv", "-q", VENV_DIR]) - - with chdir(DIST_DIR): - print("Install wheel into virtualenv...") - # lxml... - if platform.system() == "Windows" and sys.version_info[0] == 3: - subprocess.check_call( - [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"] - ) - subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()]) - - print("Running tools...") - for tool in TOOLS: - tool = join(VENV_DIR, VENV_BIN, tool) - print("> %s --version" % tool) - print(subprocess.check_output([tool, "--version"]).decode()) - - print("Virtualenv available for further testing:") - print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate"))) - - @cli.command("bdist") -@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False) -@click.argument( - "pyinstaller_version", - envvar="PYINSTALLER_VERSION", - default="git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c" -) -@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1") -@click.pass_context -def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version): +def make_bdist(): """ Build a binary distribution """ @@ -216,37 +152,40 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version) if exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) - if not use_existing_wheel: - ctx.invoke(make_wheel) - - print("Installing PyInstaller and setuptools...") - subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version]) - print(subprocess.check_output([VENV_PIP, "freeze"]).decode()) + os.makedirs(DIST_DIR, exist_ok=True) - for bdist, tools in BDISTS.items(): + for bdist, tools in sorted(BDISTS.items()): with Archive(join(DIST_DIR, archive_name(bdist))) as archive: for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" # This is PyInstaller, so it messes up paths. # We need to make sure that we are in the spec folder. with chdir(PYINSTALLER_SPEC): print("Building %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") subprocess.check_call( [ - VENV_PYINSTALLER, + "pyinstaller", "--clean", "--workpath", PYINSTALLER_TEMP, "--distpath", PYINSTALLER_DIST, "--additional-hooks-dir", PYINSTALLER_HOOKS, - # PyInstaller 3.2 does not handle Python 3.5's ucrt correctly. - "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", "--onefile", "--console", "--icon", "icon.ico", # This is PyInstaller, so setting a # different log level obviously breaks it :-) # "--log-level", "WARN", - tool ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] ) # Delete the spec file - we're good without. os.remove("{}.spec".format(tool)) @@ -255,6 +194,15 @@ def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version) executable = join(PYINSTALLER_DIST, tool) if platform.system() == "Windows": executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if executable.startswith("mitmproxy_main"): + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + print("> %s --version" % executable) print(subprocess.check_output([executable, "--version"]).decode()) @@ -306,7 +254,7 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, if wheel: files.append(wheel_name()) if bdist: - for bdist in BDISTS.keys(): + for bdist in sorted(BDISTS.keys()): files.append(archive_name(bdist)) for f in files: diff --git a/release/setup.py b/release/setup.py index d784a4a5..1d60e46c 100644 --- a/release/setup.py +++ b/release/setup.py @@ -7,8 +7,6 @@ setup( install_requires=[ "click>=6.2, <7.0", "twine>=1.6.5, <1.9", - "virtualenv>=14.0.5, <15.2", - "wheel>=0.29.0, <0.30", "pysftp==0.2.8", ], entry_points={ diff --git a/release/specs/mitmproxy b/release/specs/mitmproxy_main index 59160ff7..59160ff7 100644 --- a/release/specs/mitmproxy +++ b/release/specs/mitmproxy_main diff --git a/requirements.txt b/requirements.txt index c435c956..67a02a97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5' -e .[dev,examples,contentviews] --e ./release @@ -84,6 +84,9 @@ setup( "watchdog>=0.8.3, <0.9", "brotlipy>=0.5.1, <0.7", "sortedcontainers>=1.5.4, <1.6", + # transitive from cryptography, we just blacklist here. + # https://github.com/pypa/setuptools/issues/861 + "setuptools>=11.3, !=29.0.0", ], extras_require={ ':sys_platform == "win32"': [ diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index c31f4e9b..a41f6103 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -156,8 +156,8 @@ class TestScript: sc.request(f) assert tctx.master.event_log[0][0] == "error" assert len(tctx.master.event_log[0][1].splitlines()) == 6 - assert re.search('addonscripts/error.py", line \d+, in request', tctx.master.event_log[0][1]) - assert re.search('addonscripts/error.py", line \d+, in mkerr', tctx.master.event_log[0][1]) + assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.event_log[0][1]) + assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.event_log[0][1]) assert tctx.master.event_log[0][1].endswith("ValueError: Error!\n") def test_addon(self): @@ -4,10 +4,11 @@ skipsdist = True toxworkdir={env:TOX_WORK_DIR:.tox} [testenv] +basepython = python3.5 deps = {env:CI_DEPS:} -rrequirements.txt -passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* +passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* setenv = HOME = {envtmpdir} commands = mitmdump --sysinfo @@ -29,3 +30,27 @@ commands = mitmproxy/proxy/protocol/ \ mitmproxy/log.py \ mitmproxy/tools/dump.py mitmproxy/tools/web + +[testenv:wheel] +recreate = True +deps = +commands = + python setup.py -q bdist_wheel --dist-dir release/dist + pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl + # skip `mitmproxy --version` if SKIP_MITMPROXY is defined. + {env:SKIP_MITMPROXY:mitmproxy --version} + mitmdump --version + mitmweb --version + pathod --version + pathoc --version + +[testenv:rtool] +deps = + -rrequirements.txt + -e./release + # The 3.2 release is broken 🎉 + # the next commit after this updates the bootloaders, which then segfault! 🎉 + # https://github.com/pyinstaller/pyinstaller/issues/2232 + git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c +commands = + rtool {posargs} |