aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAldo Cortesi <aldo@corte.si>2018-03-08 11:42:36 +1300
committerGitHub <noreply@github.com>2018-03-08 11:42:36 +1300
commit56174783591700871249445ecefec79e6d6d1a6a (patch)
tree48899437289ccafefa043e921ce5d1ac98899d9a
parent56904e02c36ddd9c08909ba9909f46bf223e3761 (diff)
parent0debcf5bf92b71e58ad723629e35b7f5a3393656 (diff)
downloadmitmproxy-56174783591700871249445ecefec79e6d6d1a6a.tar.gz
mitmproxy-56174783591700871249445ecefec79e6d6d1a6a.tar.bz2
mitmproxy-56174783591700871249445ecefec79e6d6d1a6a.zip
Merge pull request #2977 from mitmproxy/updocs2
Revamp CI generation of snapshots and documentation
-rw-r--r--.appveyor.yml45
-rw-r--r--.travis.yml34
-rwxr-xr-xdocs/ci13
-rwxr-xr-xdocs/upload-archive6
-rwxr-xr-xdocs/upload-master8
-rwxr-xr-xdocs/upload-stable6
-rwxr-xr-xrelease/ci.py263
-rw-r--r--release/known_hosts.enc1
-rw-r--r--release/rtool.pem68
-rwxr-xr-xrelease/rtool.py286
-rw-r--r--tox.ini23
11 files changed, 324 insertions, 429 deletions
diff --git a/.appveyor.yml b/.appveyor.yml
index 6891f1b3..90364823 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -9,22 +9,8 @@ environment:
CI_DEPS: codecov>=2.0.5
CI_COMMANDS: codecov
matrix:
- - PYTHON: "C:\\Python35"
- TOXENV: "py35"
- # TODO: ENABLE WHEN AVAILABLE
- # - PYTHON: "C:\\Python36"
- # TOXENV: "py36"
-
- SNAPSHOT_HOST:
- secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
- SNAPSHOT_PORT:
- secure: TiJPtg60/edYTH8RnoBErg==
- SNAPSHOT_USER:
- secure: 6yBwmO5gv4vAwoFYII8qjQ==
- SNAPSHOT_PASS:
- secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
- RTOOL_KEY:
- secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM
+ - PYTHON: "C:\\Python36"
+ TOXENV: "py36"
install:
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@@ -36,17 +22,10 @@ test_script:
- ps: |
$Env:VERSION = $(python -m mitmproxy.version)
$Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
- tox -e rtool -- wheel
- tox -e rtool -- bdist
-
+ tox -e cibuild -- build
- ps: |
- if(
- ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and
- (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
- ) {
- echo "Decrypt license..."
- tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
- $ibVersion = "17.12.0"
+ if($Env:RTOOL_KEY) {
+ $ibVersion = "18.2.0"
$ibSetup = "C:\projects\mitmproxy\release\installbuilder-installer.exe"
$ibCli = "C:\Program Files (x86)\BitRock InstallBuilder Enterprise $ibVersion\bin\builder-cli.exe"
if (!(Test-Path $ibSetup)) {
@@ -58,8 +37,8 @@ test_script:
}
echo "Install InstallBuilder..."
Start-Process $ibSetup "--mode unattended --unattendedmodeui none" -PassThru -NoNewWindow -Wait
- # Wait until executable exists - no idea why this is necessary.
- while (!(Test-Path $ibCli)) { Start-Sleep 0.1 }
+ echo "Decrypt license..."
+ tox -e cibuild -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml
echo "Run InstallBuilder..."
&$ibCli `
build `
@@ -73,15 +52,7 @@ test_script:
}
deploy_script:
- # we build binaries on every run, but we only upload them for master snapshots or tags.
- ps: |
- if(
- ($Env:TOXENV -match "py35") -and
- (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true"))
- ) {
- tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts
- tox -e rtool -- upload-snapshot --bdist --wheel --installer
- }
+ ps: tox -e cibuild -- upload
cache:
- C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml
diff --git a/.travis.yml b/.travis.yml
index c22038fd..66a4f61e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,12 +21,12 @@ matrix:
- os: osx
osx_image: xcode7.3
language: generic
- env: TOXENV=py35 BDIST=1
+ env: TOXENV=py36 BDIST=1
- python: 3.5
- env: TOXENV=py35 BDIST=1
+ env: TOXENV=py35
dist: precise
- python: 3.6
- env: TOXENV=py36
+ env: TOXENV=py36 BDIST=1 WHEEL=1
- python: 3.6
env: TOXENV=individual_coverage
- language: node_js
@@ -42,6 +42,17 @@ matrix:
yarn: true
directories:
- web/node_modules
+ - language: python
+ env:
+ - NAME=docs
+ install:
+ - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
+ - sudo dpkg -i hugo*.deb
+ - pip install awscli
+ script:
+ - cd docs && ./ci
+ after_success:
+ - echo done
install:
- |
@@ -50,12 +61,11 @@ install:
brew update || brew update
brew outdated pyenv || brew upgrade pyenv
eval "$(pyenv init -)"
- env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3
- pyenv global 3.5.3
- pyenv shell 3.5.3
- pip install -U pip setuptools wheel virtualenv
+ env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4
+ pyenv global 3.6.4
+ pyenv shell 3.6.4
fi
- - pip install tox
+ - pip install tox virtualenv setuptools
script:
- tox -- --verbose --cov-report=term
@@ -63,16 +73,14 @@ script:
if [[ $BDIST == "1" ]]
then
git fetch --unshallow --tags
- tox -e rtool -- bdist
+ tox -e cibuild -- build
fi
after_success:
- # we build binaries on every run, but we only upload them for master snapshots or tags.
- |
- if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+ if [[ $BDIST == "1" ]]
then
- tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts
- tox -e rtool -- upload-snapshot --bdist
+ tox -e cibuild -- upload
fi
notifications:
diff --git a/docs/ci b/docs/ci
new file mode 100755
index 00000000..1584c5e1
--- /dev/null
+++ b/docs/ci
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This script gets run from CI to render and upload docs
+
+./build
+
+# Only upload if we have defined credentials - we only have these defined for
+# trusted commits (i.e. not PRs).
+if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then
+ aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
+ aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
+ --paths "/master"
+fi
diff --git a/docs/upload-archive b/docs/upload-archive
index 5622a2ae..86dd248e 100755
--- a/docs/upload-archive
+++ b/docs/upload-archive
@@ -12,6 +12,6 @@ SPATH="/archive/$1"
aws configure set preview.cloudfront true
aws --profile mitmproxy \
s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH
-# aws --profile mitmproxy \
-# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
-# --paths "$SPATH"
+aws --profile mitmproxy \
+ cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
+ --paths "$SPATH"
diff --git a/docs/upload-master b/docs/upload-master
deleted file mode 100755
index c8fcda80..00000000
--- a/docs/upload-master
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-aws configure set preview.cloudfront true
-aws --profile mitmproxy \
- s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
-# aws --profile mitmproxy \
-# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
-# --paths "/master"
diff --git a/docs/upload-stable b/docs/upload-stable
index 7c658113..c2c1267e 100755
--- a/docs/upload-stable
+++ b/docs/upload-stable
@@ -3,6 +3,6 @@
aws configure set preview.cloudfront true
aws --profile mitmproxy \
s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable
-# aws --profile mitmproxy \
-# cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
-# --paths "/master"
+aws --profile mitmproxy \
+ cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
+ --paths "/stable"
diff --git a/release/ci.py b/release/ci.py
new file mode 100755
index 00000000..94b1f13d
--- /dev/null
+++ b/release/ci.py
@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+
+import contextlib
+import os
+import platform
+import sys
+import shutil
+import subprocess
+import tarfile
+import zipfile
+from os.path import join, abspath, dirname, exists, basename
+
+import cryptography.fernet
+import click
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if platform.system() == "Windows":
+ VENV_BIN = "Scripts"
+ PYINSTALLER_ARGS = [
+ # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+ "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+ ]
+else:
+ VENV_BIN = "bin"
+ PYINSTALLER_ARGS = []
+
+# ZipFile and tarfile have slightly different APIs. Fix that.
+if platform.system() == "Windows":
+ def Archive(name):
+ a = zipfile.ZipFile(name, "w")
+ a.add = a.write
+ return a
+else:
+ def Archive(name):
+ return tarfile.open(name, "w:gz")
+
+PLATFORM_TAG = {
+ "Darwin": "osx",
+ "Windows": "windows",
+ "Linux": "linux",
+}.get(platform.system(), platform.system())
+
+ROOT_DIR = abspath(join(dirname(__file__), ".."))
+RELEASE_DIR = join(ROOT_DIR, "release")
+
+BUILD_DIR = join(RELEASE_DIR, "build")
+DIST_DIR = join(RELEASE_DIR, "dist")
+
+PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
+PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+
+VENV_DIR = join(BUILD_DIR, "venv")
+
+# Project Configuration
+VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
+BDISTS = {
+ "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
+ "pathod": ["pathoc", "pathod"]
+}
+if platform.system() == "Windows":
+ BDISTS["mitmproxy"].remove("mitmproxy")
+
+TOOLS = [
+ tool
+ for tools in sorted(BDISTS.values())
+ for tool in tools
+]
+
+TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
+BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
+if TAG:
+ VERSION = TAG
+ UPLOAD_DIR = VERSION
+elif BRANCH:
+ VERSION = BRANCH
+ UPLOAD_DIR = "branches/%s" % VERSION
+else:
+ print("Could not establish build name - exiting." % BRANCH)
+ sys.exit(0)
+
+
+print("BUILD VERSION=%s" % VERSION)
+
+
+def archive_name(bdist: str) -> str:
+ if platform.system() == "Windows":
+ ext = "zip"
+ else:
+ ext = "tar.gz"
+ return "{project}-{version}-{platform}.{ext}".format(
+ project=bdist,
+ version=VERSION,
+ platform=PLATFORM_TAG,
+ ext=ext
+ )
+
+
+def wheel_name() -> str:
+ return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
+
+
+def installer_name() -> str:
+ ext = {
+ "Windows": "exe",
+ "Darwin": "dmg",
+ "Linux": "run"
+ }[platform.system()]
+ return "mitmproxy-{version}-{platform}-installer.{ext}".format(
+ version=VERSION,
+ platform=PLATFORM_TAG,
+ ext=ext,
+ )
+
+
+@contextlib.contextmanager
+def chdir(path: str):
+ old_dir = os.getcwd()
+ os.chdir(path)
+ yield
+ os.chdir(old_dir)
+
+
+@click.group(chain=True)
+def cli():
+ """
+ mitmproxy build tool
+ """
+ pass
+
+
+@cli.command("info")
+def info():
+ print("Version: %s" % VERSION)
+
+
+@cli.command("build")
+def build():
+ """
+ Build a binary distribution
+ """
+ if exists(PYINSTALLER_TEMP):
+ shutil.rmtree(PYINSTALLER_TEMP)
+ if exists(PYINSTALLER_DIST):
+ shutil.rmtree(PYINSTALLER_DIST)
+
+ os.makedirs(DIST_DIR, exist_ok=True)
+
+ if "WHEEL" in os.environ:
+ print("Building wheel...")
+ subprocess.check_call(
+ [
+ "python",
+ "setup.py", "-q", "bdist_wheel",
+ "--dist-dir", "release/dist",
+ ]
+ )
+
+ for bdist, tools in sorted(BDISTS.items()):
+ with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
+ for tool in tools:
+ # We can't have a folder and a file with the same name.
+ if tool == "mitmproxy":
+ tool = "mitmproxy_main"
+ # This is PyInstaller, so it messes up paths.
+ # We need to make sure that we are in the spec folder.
+ with chdir(PYINSTALLER_SPEC):
+ print("Building %s binary..." % tool)
+ excludes = []
+ if tool != "mitmweb":
+ excludes.append("mitmproxy.tools.web")
+ if tool != "mitmproxy_main":
+ excludes.append("mitmproxy.tools.console")
+
+ subprocess.check_call(
+ [
+ "pyinstaller",
+ "--clean",
+ "--workpath", PYINSTALLER_TEMP,
+ "--distpath", PYINSTALLER_DIST,
+ "--additional-hooks-dir", PYINSTALLER_HOOKS,
+ "--onefile",
+ "--console",
+ "--icon", "icon.ico",
+ # This is PyInstaller, so setting a
+ # different log level obviously breaks it :-)
+ # "--log-level", "WARN",
+ ]
+ + [x for e in excludes for x in ["--exclude-module", e]]
+ + PYINSTALLER_ARGS
+ + [tool]
+ )
+ # Delete the spec file - we're good without.
+ os.remove("{}.spec".format(tool))
+
+ # Test if it works at all O:-)
+ executable = join(PYINSTALLER_DIST, tool)
+ if platform.system() == "Windows":
+ executable += ".exe"
+
+ # Remove _main suffix from mitmproxy executable
+ if "_main" in executable:
+ shutil.move(
+ executable,
+ executable.replace("_main", "")
+ )
+ executable = executable.replace("_main", "")
+
+ print("> %s --version" % executable)
+ print(subprocess.check_output([executable, "--version"]).decode())
+
+ archive.add(executable, basename(executable))
+ print("Packed {}.".format(archive_name(bdist)))
+
+
+def is_pr():
+ if "TRAVIS_PULL_REQUEST" in os.environ:
+ if os.environ["TRAVIS_PULL_REQUEST"] == "false":
+ return False
+ return True
+ elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"):
+ return True
+ return False
+
+
+@cli.command("upload")
+def upload():
+ """
+ Upload snapshot to snapshot server
+ """
+ # This requires some explanation. The AWS access keys are only exposed to
+ # privileged builds - that is, they are not available to PRs from forks.
+ # However, they ARE exposed to PRs from a branch within the main repo. This
+ # check catches that corner case, and prevents an inadvertent upload.
+ if is_pr():
+ print("Refusing to upload a pull request")
+ return
+ if "AWS_ACCESS_KEY_ID" in os.environ:
+ subprocess.check_call(
+ [
+ "aws", "s3", "cp",
+ "--acl", "public-read",
+ DIST_DIR + "/",
+ "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
+ "--recursive",
+ ]
+ )
+
+
+@cli.command("decrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def decrypt(infile, outfile, key):
+ f = cryptography.fernet.Fernet(key.encode())
+ outfile.write(f.decrypt(infile.read()))
+
+
+if __name__ == "__main__":
+ cli()
diff --git a/release/known_hosts.enc b/release/known_hosts.enc
deleted file mode 100644
index 585ee678..00000000
--- a/release/known_hosts.enc
+++ /dev/null
@@ -1 +0,0 @@
-gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7 \ No newline at end of file
diff --git a/release/rtool.pem b/release/rtool.pem
deleted file mode 100644
index 097dff1a..00000000
--- a/release/rtool.pem
+++ /dev/null
@@ -1,68 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB
-
-0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y
-byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU
-V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z
-T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh
-PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl
-w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD
-P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt
-JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE
-yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km
-cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe
-hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8
-jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx
-QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey
-OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL
-kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR
-AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc
-sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko
-SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz
-5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P
-cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk
-kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/
-Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9
-3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4
-mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t
-8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j
-FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf
-EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP
-a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR
-AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+
-m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P
-bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q
-QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx
-uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq
-yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K
-u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK
-MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0
-8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq
-y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz
-G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi
-o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq
-2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL
-5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp
-30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac
-7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA
-1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8
-eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB
-bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO
-VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+
-2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175
------END RSA PRIVATE KEY-----
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y
-qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe
-24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX
-FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ
-0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI
-KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt
-0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+
-tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ
-TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ
-q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J
-6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD
-5I1jQDXFbjAgAnaSyqyW368CAwEAAQ==
------END PUBLIC KEY----- \ No newline at end of file
diff --git a/release/rtool.py b/release/rtool.py
index 25f37e61..0009cf89 100755
--- a/release/rtool.py
+++ b/release/rtool.py
@@ -1,79 +1,22 @@
#!/usr/bin/env python3
import contextlib
-import fnmatch
import os
import sys
import platform
-import re
import runpy
import shlex
-import shutil
import subprocess
-import tarfile
-import zipfile
-from os.path import join, abspath, dirname, exists, basename
+from os.path import join, abspath, dirname
-import click
import cryptography.fernet
-import pysftp
-
-# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
-# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
-if platform.system() == "Windows":
- VENV_BIN = "Scripts"
- PYINSTALLER_ARGS = [
- # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
- "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
- ]
-else:
- VENV_BIN = "bin"
- PYINSTALLER_ARGS = []
-
-# ZipFile and tarfile have slightly different APIs. Fix that.
-if platform.system() == "Windows":
- def Archive(name):
- a = zipfile.ZipFile(name, "w")
- a.add = a.write
- return a
-else:
- def Archive(name):
- return tarfile.open(name, "w:gz")
+import click
-PLATFORM_TAG = {
- "Darwin": "osx",
- "Windows": "windows",
- "Linux": "linux",
-}.get(platform.system(), platform.system())
ROOT_DIR = abspath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
-
-BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")
-
-PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
-# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
-PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
-PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
-PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
-
-VENV_DIR = join(BUILD_DIR, "venv")
-
-# Project Configuration
VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-BDISTS = {
- "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
- "pathod": ["pathoc", "pathod"]
-}
-if platform.system() == "Windows":
- BDISTS["mitmproxy"].remove("mitmproxy")
-
-TOOLS = [
- tool
- for tools in sorted(BDISTS.values())
- for tool in tools
-]
def git(args: str) -> str:
@@ -86,50 +29,12 @@ def get_version(dev: bool = False, build: bool = False) -> str:
return x["get_version"](dev, build, True)
-def set_version(dev: bool) -> None:
- """
- Update version information in mitmproxy's version.py to either include hardcoded information or not.
- """
- version = get_version(dev, dev)
- with open(VERSION_FILE, "r") as f:
- content = f.read()
- content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M)
- with open(VERSION_FILE, "w") as f:
- f.write(content)
-
-
-def archive_name(bdist: str) -> str:
- if platform.system() == "Windows":
- ext = "zip"
- else:
- ext = "tar.gz"
- return "{project}-{version}-{platform}.{ext}".format(
- project=bdist,
- version=get_version(),
- platform=PLATFORM_TAG,
- ext=ext
- )
-
-
def wheel_name() -> str:
return "mitmproxy-{version}-py3-none-any.whl".format(
version=get_version(True),
)
-def installer_name() -> str:
- ext = {
- "Windows": "exe",
- "Darwin": "dmg",
- "Linux": "run"
- }[platform.system()]
- return "mitmproxy-{version}-{platform}-installer.{ext}".format(
- version=get_version(),
- platform=PLATFORM_TAG,
- ext=ext,
- )
-
-
@contextlib.contextmanager
def chdir(path: str):
old_dir = os.getcwd()
@@ -146,24 +51,6 @@ def cli():
pass
-@cli.command("encrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def encrypt(infile, outfile, key):
- f = cryptography.fernet.Fernet(key.encode())
- outfile.write(f.encrypt(infile.read()))
-
-
-@cli.command("decrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def decrypt(infile, outfile, key):
- f = cryptography.fernet.Fernet(key.encode())
- outfile.write(f.decrypt(infile.read()))
-
-
@cli.command("contributors")
def contributors():
"""
@@ -176,97 +63,6 @@ def contributors():
f.write(contributors_data.encode())
-@cli.command("wheel")
-def make_wheel():
- """
- Build a Python wheel
- """
- set_version(True)
- try:
- subprocess.check_call([
- "tox", "-e", "wheel",
- ], env={
- **os.environ,
- "VERSION": get_version(True),
- })
- finally:
- set_version(False)
-
-
-@cli.command("bdist")
-def make_bdist():
- """
- Build a binary distribution
- """
- if exists(PYINSTALLER_TEMP):
- shutil.rmtree(PYINSTALLER_TEMP)
- if exists(PYINSTALLER_DIST):
- shutil.rmtree(PYINSTALLER_DIST)
-
- os.makedirs(DIST_DIR, exist_ok=True)
-
- for bdist, tools in sorted(BDISTS.items()):
- with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
- for tool in tools:
- # We can't have a folder and a file with the same name.
- if tool == "mitmproxy":
- tool = "mitmproxy_main"
- # This is PyInstaller, so it messes up paths.
- # We need to make sure that we are in the spec folder.
- with chdir(PYINSTALLER_SPEC):
- print("Building %s binary..." % tool)
- excludes = []
- if tool != "mitmweb":
- excludes.append("mitmproxy.tools.web")
- if tool != "mitmproxy_main":
- excludes.append("mitmproxy.tools.console")
-
- # Overwrite mitmproxy/version.py to include commit info
- set_version(True)
- try:
- subprocess.check_call(
- [
- "pyinstaller",
- "--clean",
- "--workpath", PYINSTALLER_TEMP,
- "--distpath", PYINSTALLER_DIST,
- "--additional-hooks-dir", PYINSTALLER_HOOKS,
- "--onefile",
- "--console",
- "--icon", "icon.ico",
- # This is PyInstaller, so setting a
- # different log level obviously breaks it :-)
- # "--log-level", "WARN",
- ]
- + [x for e in excludes for x in ["--exclude-module", e]]
- + PYINSTALLER_ARGS
- + [tool]
- )
- finally:
- set_version(False)
- # Delete the spec file - we're good without.
- os.remove("{}.spec".format(tool))
-
- # Test if it works at all O:-)
- executable = join(PYINSTALLER_DIST, tool)
- if platform.system() == "Windows":
- executable += ".exe"
-
- # Remove _main suffix from mitmproxy executable
- if "_main" in executable:
- shutil.move(
- executable,
- executable.replace("_main", "")
- )
- executable = executable.replace("_main", "")
-
- print("> %s --version" % executable)
- print(subprocess.check_output([executable, "--version"]).decode())
-
- archive.add(executable, basename(executable))
- print("Packed {}.".format(archive_name(bdist)))
-
-
@cli.command("upload-release")
@click.option('--username', prompt=True)
@click.password_option(confirmation_prompt=False)
@@ -305,77 +101,13 @@ def homebrew_pr():
])
-@cli.command("upload-snapshot")
-@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True)
-@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22)
-@click.option("--user", envvar="SNAPSHOT_USER", prompt=True)
-@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem"))
-@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True)
-@click.option("--wheel/--no-wheel", default=False)
-@click.option("--bdist/--no-bdist", default=False)
-@click.option("--installer/--no-installer", default=False)
-def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer):
- """
- Upload snapshot to snapshot server
- """
- cnopts = pysftp.CnOpts(
- knownhosts=join(RELEASE_DIR, 'known_hosts')
- )
- with pysftp.Connection(host=host,
- port=port,
- username=user,
- private_key=private_key,
- private_key_pass=private_key_password,
- cnopts=cnopts) as sftp:
- dir_name = "snapshots/v{}".format(get_version())
- sftp.makedirs(dir_name)
- with sftp.cd(dir_name):
- files = []
- if wheel:
- files.append(wheel_name())
- if bdist:
- for bdist in sorted(BDISTS.keys()):
- files.append(archive_name(bdist))
- if installer:
- files.append(installer_name())
-
- for f in files:
- local_path = join(DIST_DIR, f)
- remote_filename = re.sub(
- r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()),
- get_version(True, True),
- f
- )
- symlink_path = "../{}".format(f.replace(get_version(), "latest"))
-
- # Upload new version
- print("Uploading {} as {}...".format(f, remote_filename))
- with click.progressbar(length=os.stat(local_path).st_size) as bar:
- # We hide the file during upload
- sftp.put(
- local_path,
- "." + remote_filename,
- callback=lambda done, total: bar.update(done - bar.pos)
- )
-
- # Delete old versions
- old_version = f.replace(get_version(), "*")
- for f_old in sftp.listdir():
- if fnmatch.fnmatch(f_old, old_version):
- print("Removing {}...".format(f_old))
- sftp.remove(f_old)
-
- # Show new version
- sftp.rename("." + remote_filename, remote_filename)
-
- # update symlink for the latest release
- if sftp.lexists(symlink_path):
- print("Removing {}...".format(symlink_path))
- sftp.remove(symlink_path)
- if f != wheel_name():
- # "latest" isn't a proper wheel version, so this could not be installed.
- # https://github.com/mitmproxy/mitmproxy/issues/1065
- sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
+@cli.command("encrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def encrypt(infile, outfile, key):
+ f = cryptography.fernet.Fernet(key.encode())
+ outfile.write(f.encrypt(infile.read()))
if __name__ == "__main__":
diff --git a/tox.ini b/tox.ini
index e90455c1..2d0fd047 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,27 +32,12 @@ deps =
commands =
python test/individual_coverage.py
-[testenv:wheel]
-recreate = True
-deps =
-commands =
- python setup.py -q bdist_wheel --dist-dir release/dist
- pip install {posargs} release/dist/mitmproxy-{env:VERSION:}-py3-none-any.whl
- # skip `mitmproxy --version` if SKIP_MITMPROXY is defined.
- {env:SKIP_MITMPROXY:mitmproxy --version}
- mitmdump --version
- mitmweb --version
- pathod --version
- pathoc --version
-
-[testenv:rtool]
-passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY
+[testenv:cibuild]
+passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
deps =
-rrequirements.txt
pyinstaller==3.3.1
- twine==1.9.1
- pysftp==0.2.9
-
+ awscli
commands =
mitmdump --version
- python ./release/rtool.py {posargs}
+ python ./release/ci.py {posargs}