diff options
| -rw-r--r-- | .appveyor.yml | 38 | ||||
| -rw-r--r-- | .gitattributes | 1 | ||||
| -rw-r--r-- | .github/workflows/main.yml | 210 | ||||
| -rw-r--r-- | .travis.yml | 103 | ||||
| -rwxr-xr-x | docs/ci | 2 | ||||
| -rwxr-xr-x | release/cibuild.py | 167 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf.py | 6 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf_data/protobuf01.bin (renamed from test/mitmproxy/contentviews/test_protobuf_data/protobuf01) | 0 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded.bin (renamed from test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded) | 0 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf_data/protobuf02.bin (renamed from test/mitmproxy/contentviews/test_protobuf_data/protobuf02) | bin | 213 -> 213 bytes | |||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded.bin (renamed from test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded) | 0 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_protobuf_data/protobuf03.bin (renamed from test/mitmproxy/contentviews/test_protobuf_data/protobuf03) | 0 | ||||
| -rw-r--r-- | test/mitmproxy/data/dumpfile-010.bin (renamed from test/mitmproxy/data/dumpfile-010) | bin | 2140 -> 2140 bytes | |||
| -rw-r--r-- | test/mitmproxy/data/dumpfile-011.bin (renamed from test/mitmproxy/data/dumpfile-011) | 0 | ||||
| -rw-r--r-- | test/mitmproxy/data/dumpfile-018.bin (renamed from test/mitmproxy/data/dumpfile-018) | 0 | ||||
| -rw-r--r-- | test/mitmproxy/io/test_compat.py | 6 | ||||
| -rw-r--r-- | test/mitmproxy/proxy/test_config.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/test_proxy.py | 15 | ||||
| -rw-r--r-- | test/release/test_cibuild.py | 64 | ||||
| -rw-r--r-- | tox.ini | 9 |
20 files changed, 388 insertions, 235 deletions
diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index e9c62ba1..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: '{build}' -build: off # Not a C# project - -branches: - except: - - requires-io-master - -environment: - CI_DEPS: codecov>=2.0.5 - CI_COMMANDS: codecov - matrix: - - PYTHON: "C:\\Python36" - TOXENV: "py36" - PYINSTALLER: "1" - WININSTALLER: "1" - - PYTHON: "C:\\Python37" - TOXENV: "py37" - -install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "python -m pip install --disable-pip-version-check -U pip" - - "pip install -U tox" - -test_script: - - ps: | - if ($env:APPVEYOR_REPO_COMMIT_MESSAGE.Contains("[notest]")) { - echo "!!!! Skipping tests." - } else { - tox -- --verbose --cov-report=term - } - - ps: tox -e cibuild -- build - -deploy_script: - ps: tox -e cibuild -- upload - -cache: - - C:\projects\mitmproxy\release\installbuilder\setup -> .appveyor.yml - - C:\Users\appveyor\AppData\Local\pip\cache diff --git a/.gitattributes b/.gitattributes index 69d68b8e..7e6d229f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ mitmproxy/tools/web/static/**/* -diff linguist-vendored web/src/js/filt/filt.js -diff +*.bin binary
\ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..c9f83e4e --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,210 @@ +name: CI + +on: [push, pull_request] + +# We currently use Python 3.7 for most things: +# - zstandard currently doesn't have 3.8 wheels, +# - we need to upgrade cryptography from version 2.4, which also doesn't have wheels + +env: + # Codecov + CODECOV_TOKEN: "0409bdfd-57a4-477d-a8af-f6172ef431d3" + +jobs: + lint-pr: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: TrueBrain/actions-flake8@v1.2 + lint-local: + # do not use external action when secrets are exposed. + if: github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install flake8 + - run: flake8 mitmproxy pathod examples test release + mypy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + - run: pip install mypy + - run: mypy . + test: + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - run: printenv + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - run: tox -e py37 + # codecov's GitHub action only supports Linux. https://github.com/codecov/codecov-action/issues/7 + # codecov's Python uploader has no github actions support yet. https://github.com/codecov/codecov-python/pull/214 + - name: Extract branch name # https://stackoverflow.com/a/58035262/934719 + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - run: pip install codecov + - run: > + codecov -f coverage.xml + --name python-${{ matrix.os }} + --commit ${{ github.sha }} + --slug ${{ github.repository }} + --branch ${{ steps.extract_branch.outputs.branch }} + test-py35: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.5' + - run: pip install tox + - run: tox -e py35 + build-wheel: + runs-on: ubuntu-latest + env: + CI_BUILD_WHEEL: 1 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - run: tox -e cibuild -- build + - uses: actions/upload-artifact@master + with: + name: wheel + path: release/dist + build-binaries: + strategy: + fail-fast: false + matrix: + # Old Ubuntu version for old glibc + os: [macos-latest, windows-latest, ubuntu-16.04] + runs-on: ${{ matrix.os }} + env: + CI_BUILD_PYINSTALLER: 1 + CI_BUILD_WININSTALLER: ${{ matrix.os == 'windows-latest' }} + CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - if: matrix.os == 'windows-latest' + uses: actions/cache@v1 + with: + path: release/installbuilder/setup + key: installbuilder + - run: pip install tox + - run: tox -e cibuild -- build + # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24 + - uses: actions/upload-artifact@master + with: + name: binaries.${{ matrix.os }} + path: release/dist + + test-web-ui: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: git rev-parse --abbrev-ref HEAD + - uses: actions/setup-node@v1 + - id: yarn-cache + run: echo "::set-output name=dir::$(yarn cache dir)" + - uses: actions/cache@v1 + with: + path: ${{ steps.yarn-cache.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + - working-directory: ./web + run: yarn + - working-directory: ./web + run: npm test + - run: bash <(curl -s https://codecov.io/bash) + + docs: + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - run: | + wget https://github.com/gohugoio/hugo/releases/download/v0.59.1/hugo_0.59.1_Linux-64bit.deb + sudo dpkg -i hugo*.deb + - run: tox -e docs + + # Separate from everything else because slow. + build-and-deploy-docker: + if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' + needs: [test, test-web-ui, build-wheel] + runs-on: ubuntu-latest + env: + CI_BUILD_DOCKER: 1 + DOCKER_USERNAME: mitmbot + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + - run: pip install tox + - uses: actions/download-artifact@master + with: + name: wheel + path: release/dist + - run: tox -e cibuild -- build + - run: tox -e cibuild -- upload + + deploy: + if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push' + runs-on: ubuntu-latest + needs: [test, test-web-ui, build-wheel, build-binaries] + env: + CI_BUILD_WHEEL: 1 + CI_BUILD_PYINSTALLER: 1 + CI_BUILD_WININSTALLER: 1 + TWINE_USERNAME: mitmproxy + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + # artifacts must be downloaded individually, see https://github.com/actions/download-artifact/issues/6 + - uses: actions/download-artifact@master + with: + name: wheel + path: release/dist + - uses: actions/download-artifact@master + with: + name: binaries.windows-latest + path: release/dist + - uses: actions/download-artifact@master + with: + name: binaries.macos-latest + path: release/dist + - uses: actions/download-artifact@master + with: + name: binaries.ubuntu-16.04 + path: release/dist + - run: ls release/dist + - run: pip install tox + - run: tox -e cibuild -- upload diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 035efb79..00000000 --- a/.travis.yml +++ /dev/null @@ -1,103 +0,0 @@ -language: python - -branches: - except: - - requires-io-master - -env: - global: - - CI_DEPS=codecov>=2.0.5 - - CI_COMMANDS=codecov - -git: - depth: 10000 - -matrix: - fast_finish: true - include: - - python: 3.5 - env: TOXENV=py35 # this just makes sure that our version detection shows an appropriate error message - - python: 3.6 - env: TOXENV=lint - - os: osx - osx_image: xcode7.3 - language: generic - env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1 - - python: 3.6 - env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1 WHEEL=1 DOCKER=1 - sudo: required - services: - - docker - - python: 3.6 - env: TOXENV=individual_coverage - - python: 3.7 - env: TOXENV=py37 - dist: xenial - - language: node_js - node_js: "node" - before_install: - - curl -o- -L https://yarnpkg.com/install.sh | bash - - export PATH=$HOME/.yarn/bin:$PATH - install: - - cd web && yarn - - yarn global add codecov - script: npm test && codecov - cache: - yarn: true - directories: - - web/node_modules - - python: 3.6 - env: NAME=docs TOXENV=docs - install: - - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - - sudo dpkg -i hugo*.deb - - pip install -U tox virtualenv setuptools - script: - - tox - after_success: - - echo done - -install: - - | - if [[ $TRAVIS_OS_NAME == "osx" ]] - then - brew update || brew update - brew outdated pyenv || brew upgrade pyenv - eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.9 - pyenv global 3.6.9 - pyenv shell 3.6.9 - fi - - pip install -U tox virtualenv setuptools - -script: - # All these steps MUST succeed for the job to be successful! - # Using the after_success block DOES NOT capture errors. - # Pull requests might break our build - we need to check this. - # Pull requests are not allowed to upload build artifacts - enforced by cibuild script. - - | - if [[ $TRAVIS_COMMIT_MESSAGE = *"[notest]"* ]]; then - echo "!!!! Skipping tests." - else - tox -- --verbose --cov-report=term - fi - - - | - if [[ $CIBUILD == "1" ]] - then - git fetch --unshallow --tags - tox -e cibuild -- build && tox -e cibuild -- upload - fi - -notifications: - slack: - -rooms: - mitmproxy:YaDGC9Gt9TEM7o8zkC2OLNsu - on_success: change - on_failure: change - on_start: never - -cache: - directories: - - $HOME/.pyenv - - $HOME/.cache/pip @@ -7,7 +7,7 @@ set -e # Only upload if we have defined credentials - we only have these defined for # trusted commits (i.e. not PRs). -if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then +if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $GITHUB_REF == "refs/heads/master" ]]; then aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \ --paths "/master/*" diff --git a/release/cibuild.py b/release/cibuild.py index 799c7929..46066099 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -14,9 +14,10 @@ import urllib.request import zipfile import click -import cryptography.fernet import parver +import cryptography.fernet + @contextlib.contextmanager def chdir(path: str): # pragma: no cover @@ -30,6 +31,14 @@ class BuildError(Exception): pass +def bool_from_env(envvar: str) -> bool: + val = os.environ.get(envvar, "") + if not val or val.lower() in ("0", "false"): + return False + else: + return True + + class BuildEnviron: PLATFORM_TAGS = { "Darwin": "osx", @@ -38,25 +47,27 @@ class BuildEnviron: } def __init__( - self, - *, - system="", - root_dir="", - travis_tag="", - travis_branch="", - travis_pull_request="", - appveyor_repo_tag_name="", - appveyor_repo_branch="", - appveyor_pull_request_number="", - should_build_wheel=False, - should_build_docker=False, - should_build_pyinstaller=False, - should_build_wininstaller=False, - has_aws_creds=False, - has_twine_creds=False, - docker_username="", - docker_password="", - rtool_key="", + self, + *, + system="", + root_dir="", + travis_tag="", + travis_branch="", + travis_pull_request="", + appveyor_repo_tag_name="", + appveyor_repo_branch="", + appveyor_pull_request_number="", + github_ref="", + github_event_name="", + should_build_wheel=False, + should_build_docker=False, + should_build_pyinstaller=False, + should_build_wininstaller=False, + has_aws_creds=False, + has_twine_creds=False, + docker_username="", + docker_password="", + build_key="", ): self.system = system self.root_dir = root_dir @@ -80,11 +91,14 @@ class BuildEnviron: self.appveyor_repo_branch = appveyor_repo_branch self.appveyor_pull_request_number = appveyor_pull_request_number + self.github_ref = github_ref + self.github_event_name = github_event_name + self.has_aws_creds = has_aws_creds self.has_twine_creds = has_twine_creds self.docker_username = docker_username self.docker_password = docker_password - self.rtool_key = rtool_key + self.build_key = build_key @classmethod def from_env(cls): @@ -96,19 +110,18 @@ class BuildEnviron: travis_pull_request=os.environ.get("TRAVIS_PULL_REQUEST"), appveyor_repo_tag_name=os.environ.get("APPVEYOR_REPO_TAG_NAME", ""), appveyor_repo_branch=os.environ.get("APPVEYOR_REPO_BRANCH", ""), - appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"), - should_build_wheel="WHEEL" in os.environ, - should_build_pyinstaller="PYINSTALLER" in os.environ, - should_build_wininstaller="WININSTALLER" in os.environ, - should_build_docker="DOCKER" in os.environ, - has_aws_creds="AWS_ACCESS_KEY_ID" in os.environ, - has_twine_creds=( - "TWINE_USERNAME" in os.environ and - "TWINE_PASSWORD" in os.environ - ), - docker_username=os.environ.get("DOCKER_USERNAME"), - docker_password=os.environ.get("DOCKER_PASSWORD"), - rtool_key=os.environ.get("RTOOL_KEY"), + appveyor_pull_request_number=os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER", ""), + github_ref=os.environ.get("GITHUB_REF", ""), + github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""), + should_build_wheel=bool_from_env("CI_BUILD_WHEEL"), + should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"), + should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"), + should_build_docker=bool_from_env("CI_BUILD_DOCKER"), + has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"), + has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"), + docker_username=os.environ.get("DOCKER_USERNAME", ""), + docker_password=os.environ.get("DOCKER_PASSWORD", ""), + build_key=os.environ.get("CI_BUILD_KEY", ""), ) def archive(self, path): @@ -143,26 +156,34 @@ class BuildEnviron: return ret @property - def branch(self): - return self.travis_branch or self.appveyor_repo_branch + def branch(self) -> str: + if self.travis_branch: + return self.travis_branch + if self.appveyor_repo_branch: + return self.appveyor_repo_branch + if self.github_ref and self.github_ref.startswith("refs/heads/"): + return self.github_ref.replace("refs/heads/", "") + if self.github_ref and self.github_ref.startswith("refs/pull/"): + return "pr-" + self.github_ref.split("/")[2] + return "" @property - def build_dir(self): + def build_dir(self) -> str: return os.path.join(self.release_dir, "build") @property - def dist_dir(self): + def dist_dir(self) -> str: return os.path.join(self.release_dir, "dist") @property - def docker_tag(self): + def docker_tag(self) -> str: if self.branch == "master": t = "dev" else: t = self.version return "mitmproxy/mitmproxy:{}".format(t) - def dump_info(self, fp=sys.stdout): + def dump_info(self, fp=sys.stdout) -> None: lst = [ "version", "tag", @@ -176,7 +197,9 @@ class BuildEnviron: "upload_dir", "should_build_wheel", "should_build_pyinstaller", + "should_build_wininstaller", "should_build_docker", + "should_upload_aws", "should_upload_docker", "should_upload_pypi", ] @@ -232,6 +255,8 @@ class BuildEnviron: @property def is_pull_request(self) -> bool: + if self.github_event_name == "pull_request": + return True if self.appveyor_pull_request_number: return True if self.travis_pull_request and self.travis_pull_request != "false": @@ -239,13 +264,13 @@ class BuildEnviron: return False @property - def platform_tag(self): + def platform_tag(self) -> str: if self.system in self.PLATFORM_TAGS: return self.PLATFORM_TAGS[self.system] raise BuildError("Unsupported platform: %s" % self.system) @property - def release_dir(self): + def release_dir(self) -> str: return os.path.join(self.root_dir, "release") @property @@ -257,6 +282,13 @@ class BuildEnviron: ]) @property + def should_upload_aws(self) -> bool: + return all([ + self.has_aws_creds, + (self.should_build_wheel or self.should_build_pyinstaller or self.should_build_wininstaller), + ]) + + @property def should_upload_pypi(self) -> bool: return all([ self.is_prod_release, @@ -265,18 +297,24 @@ class BuildEnviron: ]) @property - def tag(self): - return self.travis_tag or self.appveyor_repo_tag_name + def tag(self) -> str: + if self.travis_tag: + return self.travis_tag + if self.appveyor_repo_tag_name: + return self.appveyor_repo_tag_name + if self.github_ref and self.github_ref.startswith("refs/tags/"): + return self.github_ref.replace("refs/tags/", "") + return "" @property - def upload_dir(self): + def upload_dir(self) -> str: if self.tag: return self.version else: return "branches/%s" % self.version @property - def version(self): + def version(self) -> str: if self.tag: if self.tag.startswith("v"): try: @@ -300,13 +338,14 @@ def build_wheel(be: BuildEnviron): # pragma: no cover "bdist_wheel", "--dist-dir", be.dist_dir, ]) - whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] + whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl')) click.echo("Found wheel package: {}".format(whl)) subprocess.check_call(["tox", "-e", "wheeltest", "--", whl]) return whl -def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover +def build_docker_image(be: BuildEnviron): # pragma: no cover + whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl')) click.echo("Building Docker images...") subprocess.check_call([ "docker", @@ -410,22 +449,25 @@ def build_pyinstaller(be: BuildEnviron): # pragma: no cover def build_wininstaller(be: BuildEnviron): # pragma: no cover + if not be.build_key: + click.echo("Cannot build windows installer without secret key.") + return click.echo("Building wininstaller package...") - IB_VERSION = "18.8.0" + IB_VERSION = "19.10.0" IB_DIR = pathlib.Path(be.release_dir) / "installbuilder" IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe" IB_CLI = fr"C:\Program Files (x86)\BitRock InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe" IB_LICENSE = IB_DIR / "license.xml" - if True or not os.path.isfile(IB_CLI): + if not os.path.isfile(IB_CLI): if not os.path.isfile(IB_SETUP): click.echo("Downloading InstallBuilder...") def report(block, blocksize, total): done = block * blocksize if round(100 * done / total) != round(100 * (done - blocksize) / total): - click.secho(f"Downloading... {round(100*done/total)}%") + click.secho(f"Downloading... {round(100 * done / total)}%") urllib.request.urlretrieve( f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe", @@ -435,14 +477,13 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover shutil.move(IB_SETUP.with_suffix(".tmp"), IB_SETUP) click.echo("Install InstallBuilder...") - subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], - check=True) + subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True) assert os.path.isfile(IB_CLI) click.echo("Decrypt InstallBuilder license...") - f = cryptography.fernet.Fernet(be.rtool_key.encode()) - with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, open(IB_LICENSE, - "wb") as outfile: + f = cryptography.fernet.Fernet(be.build_key.encode()) + with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \ + open(IB_LICENSE, "wb") as outfile: outfile.write(f.decrypt(infile.read())) click.echo("Run InstallBuilder...") @@ -479,13 +520,12 @@ def build(): # pragma: no cover os.makedirs(be.dist_dir, exist_ok=True) if be.should_build_wheel: - whl = build_wheel(be) - # Docker image requires wheels - if be.should_build_docker: - build_docker_image(be, whl) + build_wheel(be) + if be.should_build_docker: + build_docker_image(be) if be.should_build_pyinstaller: build_pyinstaller(be) - if be.should_build_wininstaller and be.rtool_key: + if be.should_build_wininstaller: build_wininstaller(be) @@ -499,12 +539,15 @@ def upload(): # pragma: no cover Pushes the Docker image to Docker Hub. """ be = BuildEnviron.from_env() + be.dump_info() if be.is_pull_request: click.echo("Refusing to upload artifacts from a pull request!") return - if be.has_aws_creds: + if be.should_upload_aws: + num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)]) + click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...") subprocess.check_call([ "aws", "s3", "cp", "--acl", "public-read", diff --git a/test/mitmproxy/contentviews/test_protobuf.py b/test/mitmproxy/contentviews/test_protobuf.py index 791045e7..f0a91fd1 100644 --- a/test/mitmproxy/contentviews/test_protobuf.py +++ b/test/mitmproxy/contentviews/test_protobuf.py @@ -8,7 +8,7 @@ datadir = "mitmproxy/contentviews/test_protobuf_data/" def test_view_protobuf_request(tdata): v = full_eval(protobuf.ViewProtobuf()) - p = tdata.path(datadir + "protobuf01") + p = tdata.path(datadir + "protobuf01.bin") with open(p, "rb") as f: raw = f.read() @@ -19,12 +19,12 @@ def test_view_protobuf_request(tdata): v(b'foobar') -@pytest.mark.parametrize("filename", ["protobuf02", "protobuf03"]) +@pytest.mark.parametrize("filename", ["protobuf02.bin", "protobuf03.bin"]) def test_format_pbuf(filename, tdata): path = tdata.path(datadir + filename) with open(path, "rb") as f: input = f.read() - with open(path + "-decoded") as f: + with open(path.replace(".bin", "-decoded.bin")) as f: expected = f.read() assert protobuf.format_pbuf(input) == expected diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf01 b/test/mitmproxy/contentviews/test_protobuf_data/protobuf01.bin index fbfdbff3..fbfdbff3 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf01 +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf01.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded.bin index 9be61e28..9be61e28 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02-decoded.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02 b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02.bin Binary files differindex a47c45d5..a47c45d5 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf02 +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf02.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded.bin index 3d3392e1..3d3392e1 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03-decoded.bin diff --git a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03 b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03.bin index 9fb230b3..9fb230b3 100644 --- a/test/mitmproxy/contentviews/test_protobuf_data/protobuf03 +++ b/test/mitmproxy/contentviews/test_protobuf_data/protobuf03.bin diff --git a/test/mitmproxy/data/dumpfile-010 b/test/mitmproxy/data/dumpfile-010.bin Binary files differindex 435795bf..435795bf 100644 --- a/test/mitmproxy/data/dumpfile-010 +++ b/test/mitmproxy/data/dumpfile-010.bin diff --git a/test/mitmproxy/data/dumpfile-011 b/test/mitmproxy/data/dumpfile-011.bin index 936ac0cc..936ac0cc 100644 --- a/test/mitmproxy/data/dumpfile-011 +++ b/test/mitmproxy/data/dumpfile-011.bin diff --git a/test/mitmproxy/data/dumpfile-018 b/test/mitmproxy/data/dumpfile-018.bin index 6a27b5a6..6a27b5a6 100644 --- a/test/mitmproxy/data/dumpfile-018 +++ b/test/mitmproxy/data/dumpfile-018.bin diff --git a/test/mitmproxy/io/test_compat.py b/test/mitmproxy/io/test_compat.py index 4c31e363..341906ca 100644 --- a/test/mitmproxy/io/test_compat.py +++ b/test/mitmproxy/io/test_compat.py @@ -5,7 +5,7 @@ from mitmproxy import exceptions def test_load(tdata): - with open(tdata.path("mitmproxy/data/dumpfile-011"), "rb") as f: + with open(tdata.path("mitmproxy/data/dumpfile-011.bin"), "rb") as f: flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -13,7 +13,7 @@ def test_load(tdata): def test_load_018(tdata): - with open(tdata.path("mitmproxy/data/dumpfile-018"), "rb") as f: + with open(tdata.path("mitmproxy/data/dumpfile-018.bin"), "rb") as f: flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -21,7 +21,7 @@ def test_load_018(tdata): def test_cannot_convert(tdata): - with open(tdata.path("mitmproxy/data/dumpfile-010"), "rb") as f: + with open(tdata.path("mitmproxy/data/dumpfile-010.bin"), "rb") as f: flow_reader = io.FlowReader(f) with pytest.raises(exceptions.FlowReadException): list(flow_reader.stream()) diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 1319d1a9..38a6e1ad 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -14,7 +14,7 @@ class TestProxyConfig: def test_invalid_certificate(self, tdata): opts = options.Options() - opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")] + opts.certs = [tdata.path("mitmproxy/data/dumpfile-011.bin")] with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): ProxyConfig(opts) diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index c8cf6c33..f455b0ff 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,20 +1,19 @@ import argparse import platform from unittest import mock + import pytest -from mitmproxy.tools import cmdline -from mitmproxy.tools import main from mitmproxy import options from mitmproxy.proxy import ProxyConfig -from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler from mitmproxy.proxy import config - +from mitmproxy.proxy.server import ConnectionHandler, DummyServer, ProxyServer +from mitmproxy.tools import cmdline +from mitmproxy.tools import main from ..conftest import skip_windows class MockParser(argparse.ArgumentParser): - """ argparse.ArgumentParser sys.exits() by default. Make it more testable by throwing an exception instead. @@ -53,11 +52,9 @@ class TestProcessProxyOptions: class TestProxyServer: @skip_windows - @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'], - reason='Skipping due to macOS Mojave') + @pytest.mark.skipif(platform.system() != "Linux", reason="Linux-only") def test_err(self): - # binding to 0.0.0.0:1 works without special permissions on Windows and - # macOS Mojave + # binding to 0.0.0.0:1 works without special permissions on Windows and macOS Mojave+ conf = ProxyConfig(options.Options(listen_port=1)) with pytest.raises(Exception, match="Error starting proxy server"): ProxyServer(conf) diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index cfa24e63..d4ed32b0 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -58,31 +58,60 @@ def test_buildenviron_pr(): ) assert be.is_pull_request - # Mini test for appveyor - be = cibuild.BuildEnviron( - appveyor_pull_request_number="xxxx", + +def test_ci_systems(): + appveyor = cibuild.BuildEnviron( + appveyor_pull_request_number="1234", + appveyor_repo_branch="foo", + appveyor_repo_tag_name="qux", ) - assert be.is_pull_request - assert not be.is_prod_release - assert not be.is_maintenance_branch + assert appveyor.is_pull_request + assert appveyor.branch == "foo" + assert appveyor.tag == "qux" + + travis = cibuild.BuildEnviron( + travis_pull_request="1234", + travis_branch="foo", + travis_tag="foo", + ) + assert travis.is_pull_request + assert travis.branch == "foo" + assert travis.tag == "foo" + + github = cibuild.BuildEnviron( + github_event_name="pull_request", + github_ref="refs/heads/master" + ) + assert github.is_pull_request + assert github.branch == "master" + assert github.tag == "" + + github2 = cibuild.BuildEnviron( + github_event_name="pull_request", + github_ref="refs/tags/qux" + ) + assert github2.is_pull_request + assert github2.branch == "" + assert github2.tag == "qux" def test_buildenviron_commit(): # Simulates an ordinary commit on the master branch. be = cibuild.BuildEnviron( - travis_tag="", - travis_branch="master", - travis_pull_request="false", + github_ref="refs/heads/master", + github_event_name="push", should_build_wheel=True, should_build_pyinstaller=True, should_build_docker=True, docker_username="foo", docker_password="bar", + has_aws_creds=True, ) assert be.docker_tag == "mitmproxy/mitmproxy:dev" assert be.should_upload_docker assert not be.should_upload_pypi assert be.should_upload_docker + assert be.should_upload_aws assert not be.is_prod_release assert not be.is_maintenance_branch @@ -244,3 +273,20 @@ def test_buildenviron_check_version(version, tag, ok, tmpdir): else: with pytest.raises(ValueError): be.check_version() + + +def test_bool_from_env(monkeypatch): + monkeypatch.setenv("FOO", "1") + assert cibuild.bool_from_env("FOO") + + monkeypatch.setenv("FOO", "0") + assert not cibuild.bool_from_env("FOO") + + monkeypatch.setenv("FOO", "false") + assert not cibuild.bool_from_env("FOO") + + monkeypatch.setenv("FOO", "") + assert not cibuild.bool_from_env("FOO") + + monkeypatch.delenv("FOO") + assert not cibuild.bool_from_env("FOO") @@ -5,17 +5,14 @@ toxworkdir={env:TOX_WORK_DIR:.tox} [testenv] deps = - {env:CI_DEPS:} -rrequirements.txt -passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* OPENSSL RTOOL_* setenv = HOME = {envtmpdir} commands = mitmdump --version - pytest --timeout 60 --cov-report='' \ + pytest --timeout 60 --cov-report xml \ --cov=mitmproxy --cov=pathod --cov=release \ --full-cov=mitmproxy/ --full-cov=pathod/ \ {posargs} - {env:CI_COMMANDS:python -c ""} [testenv:py35] whitelist_externals = @@ -40,7 +37,7 @@ commands = python ./test/individual_coverage.py [testenv:cibuild] -passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER +passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_* deps = -rrequirements.txt pyinstaller==3.5 @@ -62,7 +59,7 @@ commands = pathoc --version [testenv:docs] -passenv = TRAVIS_* APPVEYOR_* AWS_* +passenv = GITHUB_* AWS_* deps = -rrequirements.txt awscli |