From 40925181e9d64c727f75e89acf0205bb4b87bb3a Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 31 Dec 2019 16:03:45 +0100 Subject: re-add missing CI steps and fix linting --- .github/workflows/main.yml | 25 +++++++++++++++++++------ README.rst | 11 +++++++---- setup.py | 3 --- tox.ini | 19 +++++++++++++++---- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9f83e4e..c856523b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,22 +17,35 @@ jobs: steps: - uses: actions/checkout@v1 - uses: TrueBrain/actions-flake8@v1.2 - lint-local: - # do not use external action when secrets are exposed. + flake8: if: github.event_name == 'push' runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 - - run: pip install flake8 - - run: flake8 mitmproxy pathod examples test release + - run: pip install tox + - run: tox -e flake8 + filename_matching: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install tox + - run: tox -e filename_matching mypy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - uses: actions/setup-python@v1 - - run: pip install mypy - - run: mypy . + - run: pip install tox + - run: tox -e mypy + individual_coverage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install tox + - run: tox -e individual_coverage test: strategy: fail-fast: false diff --git a/README.rst b/README.rst index 79766a4e..cfcc3e7f 100644 --- a/README.rst +++ b/README.rst @@ -95,12 +95,15 @@ Testing ------- If you've followed the procedure above, you already have all the development -requirements installed, and you can run the full test suite with tox_: +requirements installed, and you can run the basic test suite with tox_: .. code-block:: bash - tox -e py # runs Python tests - tox -e lint # checks code style + tox -e py37 # runs Python tests + tox -e flake8 # checks code style style + tox -e mypy # checks static types + +Our CI system has additional tox environments that are run on every pull request and branch on GitHub. For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders: @@ -109,7 +112,7 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test cd test/mitmproxy/addons pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py -Pytest does not check the code style, so you want to run ``tox -e lint`` again before committing. +Pytest does not check the code style, so you want to run ``tox -e flake8`` again before committing. Please ensure that all patches are accompanied by matching changes in the test suite. The project tries to maintain 100% test coverage and enforces this strictly for some parts of the codebase. diff --git a/setup.py b/setup.py index 690a433a..77b1098c 100644 --- a/setup.py +++ b/setup.py @@ -92,9 +92,7 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.7.8,<3.8", "Flask>=1.0,<1.2", - "mypy>=0.740,<0.741", "parver>=0.1,<2.0", "pytest-asyncio>=0.10.0,<0.11", "pytest-cov>=2.7.1,<3", @@ -103,7 +101,6 @@ setup( "pytest>=5.1.3,<6", "requests>=2.9.1,<3", "tox>=3.5,<3.15", - "rstcheck>=2.2,<4.0", ], 'examples': [ "beautifulsoup4>=4.4.1,<4.9" diff --git a/tox.ini b/tox.ini index 729d529a..42da5706 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35, py36, py37, lint, individual_coverage, wheeltest, docs +envlist = py35, py36, py37, flake8, filename_matching, mypy, individual_coverage, docs skipsdist = True toxworkdir={env:TOX_WORK_DIR:.tox} @@ -22,14 +22,25 @@ deps = commands = bash -c "mitmdump --version 2>&1 | grep 'mitmproxy requires Python 3.6'" -[testenv:lint] +[testenv:flake8] +deps = flake8>=3.7.8,<3.8 commands = - mitmdump --version flake8 --jobs 8 mitmproxy pathod examples test release + +[testenv:filename_matching] +commands = python ./test/filename_matching.py - rstcheck README.rst + +[testenv:mypy] +deps = mypy>=0.761,<0.762 +commands = mypy . +[testenv:rstcheck] +deps = rstcheck>=2.2,<4.0 +commands = + rstcheck README.rst + [testenv:individual_coverage] deps = -rrequirements.txt -- cgit v1.2.3 From d4cf28697d471090b776353a42365a2f6d5ddfbd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Apr 2020 20:29:07 +0200 Subject: use codecov github action --- .github/workflows/main.yml | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 648292c8..0d234358 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,10 +2,6 @@ name: CI on: [push, pull_request] -env: - # Codecov - CODECOV_TOKEN: "0409bdfd-57a4-477d-a8af-f6172ef431d3" - jobs: lint-pr: if: github.event_name == 'pull_request' @@ -13,7 +9,7 @@ jobs: steps: - uses: actions/checkout@v1 - uses: TrueBrain/actions-flake8@v1.2 - flake8: + lint-local: if: github.event_name == 'push' runs-on: ubuntu-latest steps: @@ -21,7 +17,7 @@ jobs: - uses: actions/setup-python@v1 - run: pip install tox - run: tox -e flake8 - filename_matching: + filename-matching: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -35,7 +31,7 @@ jobs: - uses: actions/setup-python@v1 - run: pip install tox - run: tox -e mypy - individual_coverage: + individual-coverage: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 @@ -56,19 +52,11 @@ jobs: python-version: '3.8' - run: pip install tox - run: tox -e py38 - # codecov's GitHub action only supports Linux. https://github.com/codecov/codecov-action/issues/7 - # codecov's Python uploader has no github actions support yet. https://github.com/codecov/codecov-python/pull/214 - - name: Extract branch name # https://stackoverflow.com/a/58035262/934719 - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - run: pip install codecov - - run: > - codecov -f coverage.xml - --name python-${{ matrix.os }} - --commit ${{ github.sha }} - --slug ${{ github.repository }} - --branch ${{ steps.extract_branch.outputs.branch }} + - uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml + name: ${{ matrix.os }} + fail_ci_if_error: true test-py35: runs-on: ubuntu-latest steps: @@ -140,7 +128,11 @@ jobs: run: yarn - working-directory: ./web run: npm test - - run: bash <(curl -s https://codecov.io/bash) + - uses: codecov/codecov-action@v1 + with: + file: ./web/coverage/coverage-final.json + name: web + fail_ci_if_error: true docs: env: -- cgit v1.2.3 From 7dda557add07de21cc701cebc60410579ed4b811 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Apr 2020 20:49:01 +0200 Subject: re-enable codecov according to the current codecov faq, this file cannot start with a period because that causes issues. :shrug: --- .codecov.yml | 1 - codecov.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 .codecov.yml create mode 100644 codecov.yml diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index db247200..00000000 --- a/.codecov.yml +++ /dev/null @@ -1 +0,0 @@ -comment: off diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..db247200 --- /dev/null +++ b/codecov.yml @@ -0,0 +1 @@ +comment: off -- cgit v1.2.3 From b5e3f736c0c6654c3ef0d1f280a4eacdb5ca91de Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Apr 2020 22:11:52 +0200 Subject: minor improvements, tests++ --- README.rst | 9 ++++----- mitmproxy/addons/command_history.py | 6 +++--- mitmproxy/net/http/http1/read.py | 3 ++- setup.py | 1 + test/mitmproxy/addons/test_command_history.py | 13 ++++++++++--- test/mitmproxy/net/http/http1/test_read.py | 3 ++- test/mitmproxy/test_command_lexer.py | 13 +++++++++++++ 7 files changed, 35 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 26881ff9..124e4765 100644 --- a/README.rst +++ b/README.rst @@ -99,9 +99,7 @@ requirements installed, and you can run the basic test suite with tox_: .. code-block:: bash - tox -e py37 # runs Python tests - tox -e flake8 # checks code style style - tox -e mypy # checks static types + tox -e py # runs Python tests Our CI system has additional tox environments that are run on every pull request and branch on GitHub. @@ -141,11 +139,12 @@ good reason not to. This is automatically enforced on every PR. If we detect a linting error, the PR checks will fail and block merging. You can run our lint checks yourself -with the following command: +with the following commands: .. code-block:: bash - tox -e lint + tox -e flake8 + tox -e mypy # checks static types .. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg diff --git a/mitmproxy/addons/command_history.py b/mitmproxy/addons/command_history.py index 2c1bf887..d5aaa928 100644 --- a/mitmproxy/addons/command_history.py +++ b/mitmproxy/addons/command_history.py @@ -27,7 +27,7 @@ class CommandHistory: def running(self): # FIXME: We have a weird bug where the contract for configure is not followed and it is never called with # confdir or command_history as updated. - self.configure("command_history") + self.configure("command_history") # pragma: no cover def configure(self, updated): if "command_history" in updated or "confdir" in updated: @@ -36,9 +36,9 @@ class CommandHistory: self.set_filter('') def done(self): - if ctx.options.command_history and len(self.history) > self.VACUUM_SIZE: + if ctx.options.command_history and len(self.history) >= self.VACUUM_SIZE: # vacuum history so that it doesn't grow indefinitely. - history_str = "\n".join(self.history[-self.VACUUM_SIZE / 2:]) + "\n" + history_str = "\n".join(self.history[-self.VACUUM_SIZE // 2:]) + "\n" self.history_file.write_text(history_str) @command.command("commands.history.add") diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 0fc03f1b..a9585d7d 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -331,7 +331,8 @@ def _read_headers(rfile): while True: line = rfile.readline() if not line or line == b"\r\n" or line == b"\n": - break + # we do have coverage of this, but coverage.py does not detect it. + break # pragma: no cover if line[0] in b" \t": if not ret: raise exceptions.HttpSyntaxException("Invalid headers") diff --git a/setup.py b/setup.py index d2ff5f8c..07e82863 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ setup( 'dev': [ "asynctest>=0.12.0", "Flask>=1.0,<1.2", + "hypothesis>=5.8,<5.9", "parver>=0.1,<2.0", "pytest-asyncio>=0.10.0,<0.11", "pytest-cov>=2.7.1,<3", diff --git a/test/mitmproxy/addons/test_command_history.py b/test/mitmproxy/addons/test_command_history.py index df20fba7..245bbc26 100644 --- a/test/mitmproxy/addons/test_command_history.py +++ b/test/mitmproxy/addons/test_command_history.py @@ -5,15 +5,22 @@ from mitmproxy.test import taddons class TestCommandHistory: - def test_load_from_file(self, tmpdir): - commands = ['cmd1', 'cmd2', 'cmd3'] - with open(tmpdir.join('command_history'), 'w') as f: + def test_load_and_save(self, tmpdir): + history_file = tmpdir.join('command_history') + commands = ["cmd1", "cmd2", "cmd3"] + with open(history_file, 'w') as f: f.write("\n".join(commands)) ch = command_history.CommandHistory() + ch.VACUUM_SIZE = 4 with taddons.context(ch) as tctx: tctx.options.confdir = str(tmpdir) assert ch.history == commands + ch.add_command("cmd4") + ch.done() + + with open(history_file, "r") as f: + assert f.read() == "cmd3\ncmd4\n" def test_add_command(self): history = command_history.CommandHistory() diff --git a/test/mitmproxy/net/http/http1/test_read.py b/test/mitmproxy/net/http/http1/test_read.py index 127f75ba..92c94fe3 100644 --- a/test/mitmproxy/net/http/http1/test_read.py +++ b/test/mitmproxy/net/http/http1/test_read.py @@ -161,7 +161,8 @@ def test_connection_close(): def test_expected_http_body_size(): # Expect: 100-continue assert expected_http_body_size( - treq(headers=Headers(expect="100-continue", content_length="42")) + treq(headers=Headers(expect="100-continue", content_length="42")), + expect_continue_as_0=True ) == 0 # Expect: 100-continue assert expected_http_body_size( diff --git a/test/mitmproxy/test_command_lexer.py b/test/mitmproxy/test_command_lexer.py index 3f009f88..cdda1085 100644 --- a/test/mitmproxy/test_command_lexer.py +++ b/test/mitmproxy/test_command_lexer.py @@ -1,5 +1,7 @@ import pyparsing import pytest +from hypothesis import given, example +from hypothesis.strategies import text from mitmproxy import command_lexer @@ -36,3 +38,14 @@ def test_partial_quoted_string(test_input, valid): ) def test_expr(test_input, expected): assert list(command_lexer.expr.parseString(test_input, parseAll=True)) == expected + + +@given(text()) +def test_quote_unquote_cycle(s): + assert command_lexer.unquote(command_lexer.quote(s)) == s + + +@given(text()) +@example("'foo\\'") +def test_unquote_never_fails(s): + command_lexer.unquote(s) -- cgit v1.2.3