From f69972d13d9c6bfc5f59e71c1253b8d81003d064 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 19:44:25 -0700 Subject: simplify tox setup --- .travis.yml | 23 ++++++++++++++--------- requirements.txt | 1 + setup.py | 5 +++-- tox.ini | 45 +++++++++++---------------------------------- 4 files changed, 29 insertions(+), 45 deletions(-) diff --git a/.travis.yml b/.travis.yml index a01bf055..24a0a5c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,23 +10,31 @@ addons: packages: - libssl-dev +env: + global: + - TOXENV=py + - CI_DEPS=codecov>=2.0.5 + - CI_COMMANDS=codecov + matrix: fast_finish: true include: - python: 2.7 - env: TOXENV=py27-ci + env: TOXENV=py27 - python: 2.7 - env: TOXENV=py27-ci NO_ALPN=1 + env: TOXENV=py27 NO_ALPN=1 - python: 3.5 - env: TOXENV=py35-ci + env: TOXENV=py35 - python: 3.5 - env: TOXENV=py35-ci NO_ALPN=1 + env: TOXENV=py35 NO_ALPN=1 - language: generic - env: TOXENV=py27-ci os: osx osx_image: xcode7.1 git: depth: 9999999 + env: TOXENV=py27 + - python: 3.5 + env: TOXENV=lint - python: 2.7 env: TOXENV=docs allow_failures: @@ -42,10 +50,7 @@ install: fi - pip install tox -before_script: - - "tox -e lint" - -script: tox +script: tox -- --cov netlib --cov mitmproxy --cov pathod after_success: - | diff --git a/requirements.txt b/requirements.txt index 9d5d5bbb..092f834a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5' -e .[dev,examples,contentviews] -e ./release \ No newline at end of file diff --git a/setup.py b/setup.py index a430d31a..c0ad7489 100644 --- a/setup.py +++ b/setup.py @@ -110,8 +110,9 @@ setup( ], 'contentviews': [ "cssutils>=1.0.1, <1.1", - "protobuf>=2.6.1, <2.7", - "pyamf>=0.8.0, <0.9", + # TODO: Find Python 3 replacements + # "protobuf>=2.6.1, <2.7", + # "pyamf>=0.8.0, <0.9", ], 'examples': [ "beautifulsoup4>=4.4.1, <4.5", diff --git a/tox.ini b/tox.ini index 13b67ad5..38507f27 100644 --- a/tox.ini +++ b/tox.ini @@ -2,52 +2,29 @@ envlist = py27, py35, docs, lint [testenv] +usedevelop=True deps = + {env:CI_DEPS:} -rrequirements.txt - codecov>=2.0.5 -passenv = USERNAME HOME HOMEPATH LOCALAPPDATA CODECOV_TOKEN CI TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* +passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* setenv = - PY3TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py - -[testenv:py27] + TESTS = test/ + HOME = {envtmpdir} commands = - py.test -n 8 --timeout 60 [] + py.test -n 4 --timeout 60 {posargs} {env:TESTS} + {env:CI_COMMANDS:python -c ""} [testenv:py35] -# remove bash & pipe & grep hack after cryptography ships with openssl 1.1.0 -whitelist_externals = bash -commands = - bash -c 'set -o pipefail ; py.test -n 8 --timeout 60 {env:PY3TESTS:} 2>&1 | grep -v Cryptography_locking_cb' - -[testenv:py27-ci] -commands = - py.test --cov netlib --cov mitmproxy --cov pathod --timeout 60 - codecov -e TOXENV - -[testenv:py35-ci] -# remove bash & pipe & grep hack after cryptography ships with openssl 1.1.0 -whitelist_externals = bash -commands = - bash -c 'set -o pipefail ; py.test --cov netlib --cov mitmproxy --cov pathod --timeout 60 {env:PY3TESTS:} 2>&1 | grep -v Cryptography_locking_cb' - codecov -e TOXENV - -[testenv:py35-ci-win] -deps = - https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl - codecov>=2.0.5 - .[dev] -commands = - py.test --cov netlib --cov mitmproxy --cov pathod --timeout 60 {env:PY3TESTS:} - codecov -e TOXENV +setenv = + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py [testenv:docs] basepython = python2.7 whitelist_externals = make changedir = docs -setenv = SPHINXOPTS="-W" -commands = make -e html +commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] -basepython = python2.7 +basepython = python deps = flake8>=2.6.2, <3 commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test -- cgit v1.2.3 From a4fde737b1a6d87e776b07836a04cc23f6aea08d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 21:18:01 -0700 Subject: fix appveyor --- .appveyor.yml | 16 ++++++++-------- README.rst | 4 ++-- dev.bat | 16 ---------------- dev.ps1 | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 26 deletions(-) delete mode 100644 dev.bat create mode 100644 dev.ps1 diff --git a/.appveyor.yml b/.appveyor.yml index 5e487ff6..71840f1b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,9 +4,9 @@ build: off # Not a C# project environment: matrix: - PYTHON: "C:\\Python27" - TOXENV: "py27-ci" + TOXENV: "py27" - PYTHON: "C:\\Python35" - TOXENV: "py35-ci-win" + TOXENV: "py35" SNAPSHOT_HOST: secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= @@ -22,18 +22,18 @@ install: - "python -m pip install --disable-pip-version-check -U pip" - "pip install -U tox" -before_test: - - "tox -e lint" - test_script: - "tox" deploy_script: ps: | - if(($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) { + if( + ($Env:TOXENV -match "py27") -and + (($Env:APPVEYOR_REPO_BRANCH -match "simplify-tox") -or ($Env:APPVEYOR_REPO_TAG -match "true")) + ) { pip install -U virtualenv - .\dev.bat - python .\release\rtool.py bdist + .\dev.ps1 + cmd /c "python .\release\rtool.py bdist 2>&1" python .\release\rtool.py upload-snapshot --bdist } diff --git a/README.rst b/README.rst index 0a890582..52584719 100644 --- a/README.rst +++ b/README.rst @@ -60,7 +60,7 @@ here_). Then do the following: git clone https://github.com/mitmproxy/mitmproxy.git cd mitmproxy - ./dev.sh + ./dev.sh # powershell .\dev.ps1 on Windows The *dev* script will create a virtualenv environment in a directory called @@ -74,7 +74,7 @@ mitmproxy test suite: .. code-block:: text - . venv/bin/activate # venv\Scripts\activate.bat on Windows + . venv/bin/activate # venv\Scripts\activate on Windows py.test Note that the main executables for the project - ``mitmdump``, ``mitmproxy``, diff --git a/dev.bat b/dev.bat deleted file mode 100644 index fe0824e0..00000000 --- a/dev.bat +++ /dev/null @@ -1,16 +0,0 @@ -@echo off -set VENV=.\venv - -virtualenv %VENV% --always-copy -if %errorlevel% neq 0 exit /b %errorlevel% -call %VENV%\Scripts\activate.bat -if %errorlevel% neq 0 exit /b %errorlevel% -python -m pip install --disable-pip-version-check -U pip -if %errorlevel% neq 0 exit /b %errorlevel% -pip install -r requirements.txt -if %errorlevel% neq 0 exit /b %errorlevel% - -echo. -echo * Created virtualenv environment in %VENV%. -echo * Installed all dependencies into the virtualenv. -echo * Activated virtualenv environment. diff --git a/dev.ps1 b/dev.ps1 new file mode 100644 index 00000000..7f329e8f --- /dev/null +++ b/dev.ps1 @@ -0,0 +1,16 @@ +$ErrorActionPreference = "Stop" +$VENV = ".\venv" + +virtualenv $VENV --always-copy +& $VENV\Scripts\activate.ps1 + +python -m pip install --disable-pip-version-check -U pip +cmd /c "pip install -r requirements.txt 2>&1" + +echo @" + + * Created virtualenv environment in $VENV. + * Installed all dependencies into the virtualenv. + * Activated virtualenv environment. + +"@ \ No newline at end of file -- cgit v1.2.3 From e6ea365f19d05d19967c000c6b095c93ff782557 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 21:18:17 -0700 Subject: resilence Cryptography_locking_cb --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 24a0a5c6..8599badc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,7 +50,7 @@ install: fi - pip install tox -script: tox -- --cov netlib --cov mitmproxy --cov pathod +script: set -o pipefail; tox -- --cov netlib --cov mitmproxy --cov pathod 2>&1 | grep -v Cryptography_locking_cb after_success: - | -- cgit v1.2.3 From e012116c3b749eb2f786e4be5223d5994b8751af Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 22:00:50 -0700 Subject: powershell magic --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 71840f1b..dff6dc78 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -23,7 +23,7 @@ install: - "pip install -U tox" test_script: - - "tox" + - ps: "tox | Select-String -NotMatch Cryptography_locking_cb" deploy_script: ps: | -- cgit v1.2.3 From 4f7d108722212f92f1abed9933ad2060a9b10853 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 22:06:08 -0700 Subject: compile docs on python 3.5 --- .travis.yml | 2 +- setup.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8599badc..a6f75913 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,7 @@ matrix: env: TOXENV=py27 - python: 3.5 env: TOXENV=lint - - python: 2.7 + - python: 3.5 env: TOXENV=docs allow_failures: - python: pypy diff --git a/setup.py b/setup.py index c0ad7489..991efa15 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ setup( "pytest-xdist>=1.14, <1.15", "sphinx>=1.3.5, <1.5", "sphinx-autobuild>=0.5.2, <0.7", - "sphinxcontrib-documentedlist>=0.3.0, <0.4", + "sphinxcontrib-documentedlist>=0.4.0, <0.5", "sphinx_rtd_theme>=0.1.9, <0.2", ], 'contentviews': [ diff --git a/tox.ini b/tox.ini index 38507f27..b39d786c 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ setenv = TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py [testenv:docs] -basepython = python2.7 +basepython = python whitelist_externals = make changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html -- cgit v1.2.3 From a57ea27bd471f7c3b96e321db337531e5a30d7fe Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 22:09:12 -0700 Subject: watchdog: always use PollingObserver --- mitmproxy/script/reloader.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mitmproxy/script/reloader.py b/mitmproxy/script/reloader.py index f5470bbf..0a3f5c93 100644 --- a/mitmproxy/script/reloader.py +++ b/mitmproxy/script/reloader.py @@ -5,12 +5,10 @@ import sys from watchdog.events import RegexMatchingEventHandler -if sys.platform == 'darwin': # pragma: no cover - from watchdog.observers.polling import PollingObserver as Observer -else: - from watchdog.observers import Observer -# The OSX reloader in watchdog 0.8.3 breaks when unobserving paths. -# We use the PollingObserver instead. +from watchdog.observers.polling import PollingObserver as Observer +# We occasionally have watchdog errors on Windows, Linux and Mac when using the native observers. +# After reading through the watchdog source code and issue tracker, +# we may want to replace this with a very simple implementation of our own. _observers = {} -- cgit v1.2.3 From 7cae0c268f728a6dafede4cc04fabdad1f3e5c52 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 23:18:55 -0700 Subject: do not run tests in parallel this makes ci very unreliable, see https://travis-ci.org/mitmproxy/mitmproxy/jobs/140994186 I don't know of a way to fix the test collection bug. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b39d786c..6e3f570f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ setenv = TESTS = test/ HOME = {envtmpdir} commands = - py.test -n 4 --timeout 60 {posargs} {env:TESTS} + py.test --timeout 60 {posargs} {env:TESTS} {env:CI_COMMANDS:python -c ""} [testenv:py35] -- cgit v1.2.3 From f7d4319c5aa5ea49eefbbef0424cee074e326960 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 23:48:19 -0700 Subject: minor fixes --- .travis.yml | 1 - mitmproxy/script/reloader.py | 1 - test/mitmproxy/data/scripts/tcp_stream_modify.py | 2 +- tox.ini | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6f75913..29d0897c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ addons: env: global: - - TOXENV=py - CI_DEPS=codecov>=2.0.5 - CI_COMMANDS=codecov diff --git a/mitmproxy/script/reloader.py b/mitmproxy/script/reloader.py index 0a3f5c93..50401034 100644 --- a/mitmproxy/script/reloader.py +++ b/mitmproxy/script/reloader.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, print_function, division import os -import sys from watchdog.events import RegexMatchingEventHandler diff --git a/test/mitmproxy/data/scripts/tcp_stream_modify.py b/test/mitmproxy/data/scripts/tcp_stream_modify.py index d7953ef9..0965beba 100644 --- a/test/mitmproxy/data/scripts/tcp_stream_modify.py +++ b/test/mitmproxy/data/scripts/tcp_stream_modify.py @@ -1,4 +1,4 @@ def tcp_message(ctx, flow): message = flow.messages[-1] if not message.from_client: - message.content = message.content.replace("foo", "bar") + message.content = message.content.replace(b"foo", b"bar") diff --git a/tox.ini b/tox.ini index 6e3f570f..4ad9e967 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ commands = [testenv:py35] setenv = TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py + HOME = {envtmpdir} [testenv:docs] basepython = python -- cgit v1.2.3 From 01104b4761d651437257d9307b25f746e066a0b9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 28 Jun 2016 23:55:29 -0700 Subject: collect coverage on appveyor --- .appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index dff6dc78..57580b1a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,6 +2,8 @@ version: '{build}' build: off # Not a C# project environment: + CI_DEPS: codecov>=2.0.5 + CI_COMMANDS: codecov matrix: - PYTHON: "C:\\Python27" TOXENV: "py27" @@ -23,7 +25,7 @@ install: - "pip install -U tox" test_script: - - ps: "tox | Select-String -NotMatch Cryptography_locking_cb" + - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod | Select-String -NotMatch Cryptography_locking_cb" deploy_script: ps: | -- cgit v1.2.3 From 6181bf978dae9b9803f49c0bdcbda4988cd3e2ec Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 29 Jun 2016 00:05:03 -0700 Subject: remove superfluous statements in tox.ini --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index 4ad9e967..4700e6ca 100644 --- a/tox.ini +++ b/tox.ini @@ -20,12 +20,9 @@ setenv = HOME = {envtmpdir} [testenv:docs] -basepython = python -whitelist_externals = make changedir = docs commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] -basepython = python deps = flake8>=2.6.2, <3 commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test -- cgit v1.2.3 From b27c6ee18181d3300d5f47a3b0c1805b7e3d5c75 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 29 Jun 2016 00:06:04 -0700 Subject: appveyor: build snapshots on master --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 57580b1a..339342ae 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -31,7 +31,7 @@ deploy_script: ps: | if( ($Env:TOXENV -match "py27") -and - (($Env:APPVEYOR_REPO_BRANCH -match "simplify-tox") -or ($Env:APPVEYOR_REPO_TAG -match "true")) + (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) ) { pip install -U virtualenv .\dev.ps1 -- cgit v1.2.3