diff options
38 files changed, 402 insertions, 144 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 5cf194a9..b95b23e3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,8 @@ environment: secure: 6yBwmO5gv4vAwoFYII8qjQ== SNAPSHOT_PASS: secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV + RTOOL_KEY: + secure: 0a+UUNbA+JjquyAbda4fd0JmiwL06AdG6torRPdCvbPDbKHnaW/BHHp1nRPytOKM install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" @@ -30,6 +32,29 @@ test_script: tox -e wheel tox -e rtool -- bdist + - ps: | + if( + ($Env:TOXENV -match "py35") -and + (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) + ) { + tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml + if (!(Test-Path "C:\projects\mitmproxy\release\installbuilder-installer.exe")) { + "Download InstallBuilder..." + (New-Object System.Net.WebClient).DownloadFile( + "https://installbuilder.bitrock.com/installbuilder-enterprise-16.11.1-windows-installer.exe", + "C:\projects\mitmproxy\release\installbuilder-installer.exe" + ) + } + Start-Process "C:\projects\mitmproxy\release\installbuilder-installer.exe" "--mode unattended --unattendedmodeui none" -Wait + & 'C:\Program Files (x86)\BitRock InstallBuilder Enterprise 16.11.1\bin\builder-cli.exe' ` + build ` + .\release\installbuilder\mitmproxy.xml ` + windows ` + --license .\release\installbuilder\license.xml ` + --setvars project.version=$Env:VERSION ` + --verbose + } + deploy_script: # we build binaries on every run, but we only upload them for master snapshots or tags. ps: | @@ -37,10 +62,11 @@ deploy_script: ($Env:TOXENV -match "py35") -and (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) ) { - tox -e rtool -- upload-snapshot --bdist --wheel + tox -e rtool -- upload-snapshot --bdist --wheel --installer } cache: + - C:\projects\mitmproxy\release\installbuilder-installer.exe -> .appveyor.yml - C:\Users\appveyor\AppData\Local\pip\cache notifications: diff --git a/docs/install.rst b/docs/install.rst index b6160a9c..1fe09aca 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -61,6 +61,7 @@ Installation On macOS You can use Homebrew to install everything: .. code:: bash + brew install mitmproxy Or you can download the pre-built binary packages from `mitmproxy.org`_. @@ -86,17 +87,20 @@ If you already have an older version of Python 3.5 installed, make sure to insta (pip is included in Python by default). If pip aborts with an error, make sure you are using the current version of pip. .. code:: powershell + python -m pip install --upgrade pip Next, add Python and the Python Scripts directory to your **PATH** variable. You can do this easily by running the following in powershell: .. code:: powershell + [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User") Now, you can install mitmproxy by running .. code:: powershell + pip install mitmproxy Once the installation is complete, you can run :ref:`mitmdump` from a command prompt. diff --git a/mitmproxy/__init__.py b/mitmproxy/__init__.py index e69de29b..9697de87 100644 --- a/mitmproxy/__init__.py +++ b/mitmproxy/__init__.py @@ -0,0 +1,3 @@ +# https://github.com/mitmproxy/mitmproxy/issues/1809 +# import script here so that pyinstaller registers it. +from . import script # noqa diff --git a/mitmproxy/addons/replace.py b/mitmproxy/addons/replace.py index b675b779..09200d5d 100644 --- a/mitmproxy/addons/replace.py +++ b/mitmproxy/addons/replace.py @@ -16,21 +16,22 @@ class Replace: rex: a regular expression, as bytes. s: the replacement string, as bytes """ - lst = [] - for fpatt, rex, s in options.replacements: - flt = flowfilter.parse(fpatt) - if not flt: - raise exceptions.OptionsError( - "Invalid filter pattern: %s" % fpatt - ) - try: - re.compile(rex) - except re.error as e: - raise exceptions.OptionsError( - "Invalid regular expression: %s - %s" % (rex, str(e)) - ) - lst.append((rex, s, flt)) - self.lst = lst + if "replacements" in updated: + lst = [] + for fpatt, rex, s in options.replacements: + flt = flowfilter.parse(fpatt) + if not flt: + raise exceptions.OptionsError( + "Invalid filter pattern: %s" % fpatt + ) + try: + re.compile(rex) + except re.error as e: + raise exceptions.OptionsError( + "Invalid regular expression: %s - %s" % (rex, str(e)) + ) + lst.append((rex, s, flt)) + self.lst = lst def execute(self, f): for rex, s, flt in self.lst: diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index c89fa085..93245760 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -4,6 +4,7 @@ import shlex import sys import threading import traceback +import types from mitmproxy import exceptions from mitmproxy import ctx @@ -14,19 +15,6 @@ import watchdog.events from watchdog.observers import polling -class NS: - def __init__(self, ns): - self.__dict__["ns"] = ns - - def __getattr__(self, key): - if key not in self.ns: - raise AttributeError("No such element: %s", key) - return self.ns[key] - - def __setattr__(self, key, value): - self.__dict__["ns"][key] = value - - def parse_command(command): """ Returns a (path, args) tuple. @@ -113,8 +101,8 @@ def load_script(path, args): return ns = {'__file__': os.path.abspath(path)} with scriptenv(path, args): - exec(code, ns, ns) - return NS(ns) + exec(code, ns) + return types.SimpleNamespace(**ns) class ReloadHandler(watchdog.events.FileSystemEventHandler): diff --git a/mitmproxy/addons/stickyauth.py b/mitmproxy/addons/stickyauth.py index c0d7893d..1a1d4fc4 100644 --- a/mitmproxy/addons/stickyauth.py +++ b/mitmproxy/addons/stickyauth.py @@ -8,18 +8,22 @@ class StickyAuth: self.hosts = {} def configure(self, options, updated): - if options.stickyauth: - flt = flowfilter.parse(options.stickyauth) - if not flt: - raise exceptions.OptionsError( - "stickyauth: invalid filter expression: %s" % options.stickyauth - ) - self.flt = flt + if "stickyauth" in updated: + if options.stickyauth: + flt = flowfilter.parse(options.stickyauth) + if not flt: + raise exceptions.OptionsError( + "stickyauth: invalid filter expression: %s" % options.stickyauth + ) + self.flt = flt + else: + self.flt = None def request(self, flow): - host = flow.request.host - if "authorization" in flow.request.headers: - self.hosts[host] = flow.request.headers["authorization"] - elif flowfilter.match(self.flt, flow): - if host in self.hosts: - flow.request.headers["authorization"] = self.hosts[host] + if self.flt: + host = flow.request.host + if "authorization" in flow.request.headers: + self.hosts[host] = flow.request.headers["authorization"] + elif flowfilter.match(self.flt, flow): + if host in self.hosts: + flow.request.headers["authorization"] = self.hosts[host] diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index 293de565..fb1c5072 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -34,13 +34,16 @@ class StickyCookie: self.flt = None def configure(self, options, updated): - if options.stickycookie: - flt = flowfilter.parse(options.stickycookie) - if not flt: - raise exceptions.OptionsError( - "stickycookie: invalid filter expression: %s" % options.stickycookie - ) - self.flt = flt + if "stickycookie" in updated: + if options.stickycookie: + flt = flowfilter.parse(options.stickycookie) + if not flt: + raise exceptions.OptionsError( + "stickycookie: invalid filter expression: %s" % options.stickycookie + ) + self.flt = flt + else: + self.flt = None def response(self, flow): if self.flt: diff --git a/mitmproxy/addons/streamfile.py b/mitmproxy/addons/streamfile.py index 2fc61015..5517e9dc 100644 --- a/mitmproxy/addons/streamfile.py +++ b/mitmproxy/addons/streamfile.py @@ -29,6 +29,8 @@ class StreamFile: raise exceptions.OptionsError( "Invalid filter specification: %s" % options.filtstr ) + else: + self.filt = None if "streamfile" in updated: if self.stream: self.done() diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index be761adf..25696e43 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -145,9 +145,9 @@ class View(collections.Sequence): def inbounds(self, index: int) -> bool: """ - Is this index >= 0 and < len(self) + Is this 0 <= index < len(self) """ - return index >= 0 and index < len(self) + return 0 <= index < len(self) def _rev(self, idx: int) -> int: """ @@ -362,7 +362,7 @@ class Focus: return self.view.index(self.flow) @index.setter - def index(self, idx) -> typing.Optional[int]: + def index(self, idx): if idx < 0 or idx > len(self.view) - 1: raise ValueError("Index out of view bounds") self.flow = self.view[idx] diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 78b358c9..da1a3f17 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -212,7 +212,7 @@ class OptManager(metaclass=_DefaultsMeta): if not text: return {} try: - data = ruamel.yaml.load(text, ruamel.yaml.Loader) + data = ruamel.yaml.load(text, ruamel.yaml.RoundTripLoader) except ruamel.yaml.error.YAMLError as v: snip = v.problem_mark.get_snippet() raise exceptions.OptionsError( diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 925491d7..a8f04f8d 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -451,19 +451,21 @@ def proxy_options(parser): action="store", type=int, dest="port", help="Proxy service port." ) - group.add_argument( - "--no-http2", - action="store_false", dest="http2", - help=""" - Explicitly disable HTTP/2 support. - If your OpenSSL version supports ALPN, HTTP/2 is enabled by default. - """ - ) - group.add_argument( - "--no-websocket", - action="store_false", dest="websocket", - help="Explicitly disable WebSocket support." - ) + + http2 = group.add_mutually_exclusive_group() + http2.add_argument("--http2", action="store_true", dest="http2") + http2.add_argument("--no-http2", action="store_false", dest="http2", + help="Explicitly enable/disable HTTP/2 support. " + "Disabled by default until major websites implement the spec correctly. " + "Default value will change in a future version." + ) + + websocket = group.add_mutually_exclusive_group() + websocket.add_argument("--no-websocket", action="store_false", dest="websocket", + help="Explicitly enable/disable WebSocket support. " + "Enabled by default." + ) + websocket.add_argument("--websocket", action="store_true", dest="websocket") parser.add_argument( "--upstream-auth", diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index a8e7f59f..ec637cbc 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -150,9 +150,6 @@ def save_data(path, data): def ask_save_overwrite(path, data): - if not path: - return - path = os.path.expanduser(path) if os.path.exists(path): def save_overwrite(k): if k == "y": diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index d7c312e5..fd6b3bab 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -65,6 +65,9 @@ class LogBufferBox(urwid.ListBox): self.set_focus(len(self.master.logbuffer) - 1) elif key == "g": self.set_focus(0) + elif key == "F": + o = self.master.options + o.focus_follow = not o.focus_follow return urwid.ListBox.keypress(self, size, key) @@ -355,9 +358,11 @@ class FlowListBox(urwid.ListBox): elif key == "e": self.master.toggle_eventlog() elif key == "g": - self.master.view.focus.index = 0 + if len(self.master.view): + self.master.view.focus.index = 0 elif key == "G": - self.master.view.focus.index = len(self.master.view) - 1 + if len(self.master.view): + self.master.view.focus.index = len(self.master.view) - 1 elif key == "f": signals.status_prompt.send( prompt = "Filter View", diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index ecb070d8..efeab647 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -1,23 +1,23 @@ import math import os import sys +from functools import lru_cache +from typing import Optional, Union # noqa import urwid -from mitmproxy import exceptions -from typing import Optional, Union # noqa from mitmproxy import contentviews +from mitmproxy import exceptions +from mitmproxy import export from mitmproxy import http +from mitmproxy.net.http import Headers +from mitmproxy.net.http import status_codes from mitmproxy.tools.console import common from mitmproxy.tools.console import flowdetailview from mitmproxy.tools.console import grideditor from mitmproxy.tools.console import searchable from mitmproxy.tools.console import signals from mitmproxy.tools.console import tabs -from mitmproxy import export -from mitmproxy.net.http import Headers -from mitmproxy.net.http import status_codes -from functools import lru_cache class SearchError(Exception): @@ -483,9 +483,12 @@ class FlowView(tabs.Tabs): return self._view_nextprev_flow(self.view.index(flow) - 1, flow) def change_this_display_mode(self, t): - name = contentviews.get_by_shortcut(t).name - self.view.settings[self.flow][(self.tab_offset, "prettyview")] = name - signals.flow_change.send(self, flow = self.flow) + view = contentviews.get_by_shortcut(t) + if view: + self.view.settings[self.flow][(self.tab_offset, "prettyview")] = view.name + else: + self.view.settings[self.flow][(self.tab_offset, "prettyview")] = None + signals.flow_change.send(self, flow=self.flow) def keypress(self, size, key): conn = None # type: Optional[Union[http.HTTPRequest, http.HTTPResponse]] diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index c7c05c35..1155407d 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -5,8 +5,8 @@ from mitmproxy import flowfilter from mitmproxy.addons import script from mitmproxy.tools.console import common from mitmproxy.tools.console.grideditor import base -from mitmproxy.tools.console.grideditor import col_bytes from mitmproxy.tools.console.grideditor import col_text +from mitmproxy.tools.console.grideditor import col_bytes from mitmproxy.tools.console.grideditor import col_subgrid from mitmproxy.tools.console import signals from mitmproxy.net.http import user_agents @@ -74,8 +74,8 @@ class ReplaceEditor(base.GridEditor): title = "Editing replacement patterns" columns = [ col_text.Column("Filter"), - col_bytes.Column("Regex"), - col_bytes.Column("Replacement"), + col_text.Column("Regex"), + col_text.Column("Replacement"), ] def is_error(self, col, val): @@ -94,8 +94,8 @@ class SetHeadersEditor(base.GridEditor): title = "Editing header set patterns" columns = [ col_text.Column("Filter"), - col_bytes.Column("Header"), - col_bytes.Column("Value"), + col_text.Column("Header"), + col_text.Column("Value"), ] def is_error(self, col, val): diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 73d7adbd..8afdce2c 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -35,7 +35,7 @@ from mitmproxy.utils import strutils from mitmproxy.net import tcp -EVENTLOG_SIZE = 500 +EVENTLOG_SIZE = 10000 class Logger: @@ -108,7 +108,8 @@ class ConsoleMaster(master.Master): self.logbuffer.append(e) if len(self.logbuffer) > EVENTLOG_SIZE: self.logbuffer.pop(0) - self.logbuffer.set_focus(len(self.logbuffer) - 1) + if self.options.focus_follow: + self.logbuffer.set_focus(len(self.logbuffer) - 1) def sig_call_in(self, sender, seconds, callback, args=()): def cb(*_): @@ -386,17 +387,10 @@ class ConsoleMaster(master.Master): ) def _write_flows(self, path, flows): - if not path: - return - path = os.path.expanduser(path) - try: - f = open(path, "wb") + with open(path, "wb") as f: fw = io.FlowWriter(f) for i in flows: fw.add(i) - f.close() - except IOError as v: - signals.status_message.send(message=v.strerror) def save_one_flow(self, path, flow): return self._write_flows(path, [flow]) @@ -405,8 +399,6 @@ class ConsoleMaster(master.Master): return self._write_flows(path, self.view) def load_flows_callback(self, path): - if not path: - return ret = self.load_flows_path(path) return ret or "Flows loaded from %s" % path diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 94483b3d..54876f87 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -9,6 +9,7 @@ from mitmproxy.tools.console import signals footer = [ ('heading_key', "enter/space"), ":toggle ", ('heading_key', "C"), ":clear all ", + ('heading_key', "W"), ":save ", ] @@ -17,6 +18,7 @@ def _mkhelp(): keys = [ ("enter/space", "activate option"), ("C", "clear all options"), + ("w", "save options"), ] text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) return text @@ -162,8 +164,21 @@ class Options(urwid.WidgetWrap): if key == "C": self.clearall() return None + if key == "W": + self.save() + return None return super().keypress(size, key) + def do_save(self, path): + self.master.options.save(path) + return "Saved" + + def save(self): + signals.status_prompt_path.send( + prompt = "Save options to file", + callback = self.do_save + ) + def clearall(self): self.master.options.reset() signals.update_settings.send(self) diff --git a/mitmproxy/tools/console/pathedit.py b/mitmproxy/tools/console/pathedit.py index 4447070b..10ee1416 100644 --- a/mitmproxy/tools/console/pathedit.py +++ b/mitmproxy/tools/console/pathedit.py @@ -57,8 +57,8 @@ class _PathCompleter: class PathEdit(urwid.Edit, _PathCompleter): - def __init__(self, *args, **kwargs): - urwid.Edit.__init__(self, *args, **kwargs) + def __init__(self, prompt, last_path): + urwid.Edit.__init__(self, prompt, last_path) _PathCompleter.__init__(self) def keypress(self, size, key): diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index e3424493..dce8605f 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -9,6 +9,28 @@ from mitmproxy.tools.console import signals from mitmproxy.utils import human +class PromptPath: + def __init__(self, callback, args): + self.callback, self.args = callback, args + + def __call__(self, pth): + if not pth: + return + pth = os.path.expanduser(pth) + try: + return self.callback(pth, *self.args) + except IOError as v: + signals.status_message.send(message=v.strerror) + + +class PromptStub: + def __init__(self, callback, args): + self.callback, self.args = callback, args + + def __call__(self, txt): + return self.callback(txt, *self.args) + + class ActionBar(urwid.WidgetWrap): def __init__(self): @@ -21,7 +43,8 @@ class ActionBar(urwid.WidgetWrap): self.last_path = "" - self.prompting = False + self.prompting = None + self.onekey = False self.pathprompt = False @@ -42,7 +65,7 @@ class ActionBar(urwid.WidgetWrap): def sig_prompt(self, sender, prompt, text, callback, args=()): signals.focus.send(self, section="footer") self._w = urwid.Edit(self.prep_prompt(prompt), text or "") - self.prompting = (callback, args) + self.prompting = PromptStub(callback, args) def sig_path_prompt(self, sender, prompt, callback, args=()): signals.focus.send(self, section="footer") @@ -51,7 +74,7 @@ class ActionBar(urwid.WidgetWrap): os.path.dirname(self.last_path) ) self.pathprompt = True - self.prompting = (callback, args) + self.prompting = PromptPath(callback, args) def sig_prompt_onekey(self, sender, prompt, keys, callback, args=()): """ @@ -69,7 +92,7 @@ class ActionBar(urwid.WidgetWrap): prompt.append(")? ") self.onekey = set(i[1] for i in keys) self._w = urwid.Edit(prompt, "") - self.prompting = (callback, args) + self.prompting = PromptStub(callback, args) def selectable(self): return True @@ -93,10 +116,10 @@ class ActionBar(urwid.WidgetWrap): def clear(self): self._w = urwid.Text("") - self.prompting = False + self.prompting = None def prompt_done(self): - self.prompting = False + self.prompting = None self.onekey = False self.pathprompt = False signals.status_message.send(message="") @@ -105,9 +128,9 @@ class ActionBar(urwid.WidgetWrap): def prompt_execute(self, txt): if self.pathprompt: self.last_path = txt - p, args = self.prompting + p = self.prompting self.prompt_done() - msg = p(txt, *args) + msg = p(txt) if msg: signals.status_message.send(message=msg, expire=1) diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index c68357b7..ca2c7d5d 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -21,7 +21,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: type(value) )) - if isinstance(typeinfo, typing.UnionMeta): + if typeinfo.__qualname__ == "Union": for T in typeinfo.__union_params__: try: check_type(attr_name, value, T) @@ -30,18 +30,24 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: else: return raise e - if isinstance(typeinfo, typing.TupleMeta): - check_type(attr_name, value, tuple) + elif typeinfo.__qualname__ == "Tuple": + if not isinstance(value, (tuple, list)): + raise e if len(typeinfo.__tuple_params__) != len(value): raise e for i, (x, T) in enumerate(zip(value, typeinfo.__tuple_params__)): check_type("{}[{}]".format(attr_name, i), x, T) return - if issubclass(typeinfo, typing.IO): + elif typeinfo.__qualname__ == "Sequence": + T = typeinfo.__args__[0] + if not isinstance(value, (tuple, list)): + raise e + for v in value: + check_type(attr_name, v, T) + elif typeinfo.__qualname__ == "IO": if hasattr(value, "read"): return - - if not isinstance(value, typeinfo): + elif not isinstance(value, typeinfo): raise e diff --git a/release/hooks/hook-cryptography.py b/release/hooks/hook-cryptography.py index d53a438b..5ecbd6cc 100644 --- a/release/hooks/hook-cryptography.py +++ b/release/hooks/hook-cryptography.py @@ -1,4 +1,5 @@ # Taken from the latest pyinstaller master on 2016-11-27 (0729a2b). +# flake8: noqa #----------------------------------------------------------------------------- # Copyright (c) 2005-2016, PyInstaller Development Team. @@ -40,4 +41,4 @@ cryptography_dir = os.path.dirname(get_module_file_attribute('cryptography')) for ext in EXTENSION_SUFFIXES: ffimods = glob.glob(os.path.join(cryptography_dir, '*_cffi_*%s*' % ext)) for f in ffimods: - binaries.append((f, 'cryptography'))
\ No newline at end of file + binaries.append((f, 'cryptography')) diff --git a/release/hooks/hook-pydivert.py b/release/hooks/hook-pydivert.py index 3fe8c552..72b7eb7d 100644 --- a/release/hooks/hook-pydivert.py +++ b/release/hooks/hook-pydivert.py @@ -1,3 +1,3 @@ from PyInstaller.utils.hooks import collect_data_files -datas = collect_data_files('pydivert.windivert_dll')
\ No newline at end of file +datas = collect_data_files('pydivert.windivert_dll') diff --git a/release/installbuilder/.gitignore b/release/installbuilder/.gitignore new file mode 100644 index 00000000..00c10a2d --- /dev/null +++ b/release/installbuilder/.gitignore @@ -0,0 +1,2 @@ +license.xml +*.xml.backup
\ No newline at end of file diff --git a/release/installbuilder/license.xml.enc b/release/installbuilder/license.xml.enc new file mode 100644 index 00000000..3df7210a --- /dev/null +++ b/release/installbuilder/license.xml.enc @@ -0,0 +1 @@ +gAAAAABYSXtgysddnqWL5sAreDTENJZdFkEf9p2nfMZTynY2hEep8vIDwR0P1glL40C04kLUv3piHkqyz0I893b-_YpnZC-W7qwfhChoFxnAQd0NY_d1mLsCHF1nQK0k907IbextZ3XHSCqSkS2nw0S41p9sEbzWLludSRRqJRn-AmxAKdm5s43kheNpP5iArWKtBrtubgytIDanulTfibdAPHaZedyx3JDww3hIUgo2shird7L4q2xQsE0bBDBgGrlndZ6lYGJDKf600LpOPSzSON6uXdPefhPD1bZP0GAFTNMAkm0K39e9J-j9Xz5ao7bEswfR6pcYO1uQC9VmxJYdLkuY9vYiltED6m5dW--NXVx0pudr_e5BXfpT5-RdUHaGM0Od6EZggeD5PNgUtWXxmXd6C0gFBZZEx_KobDXrCD9s8tmRJjb3ACBSCh2NdfFCCfOU3JRHeSgANTDYzY0pcJicoE5UcvP2dEICVvEORwbTumtfWbuDSbXwHqcynVjkJbwxHhRMMeJtjnx4dbb6_wdp2RjOUilTb2ob40OWyU0szgpSe2-5XS_F7ixL7vMf00OHFtTDk0Wg_dv15IR3C-zK8WFWEr4b4POc1P6pRrKOLKIFIKBGGZWF-S-u3gedAL2VLpFoUCG5FMnnJGv2iNfNYkG20zObn9K6FG5uNq6T0ZvfJIjJYNvXV45mx7jhOg2R5HAkZXlx16iIv95qlHIvJR6UZKHqWr_Hp3YvaxatkqbFwpXxQU0xtMJO3dXafATFMW_l
\ No newline at end of file diff --git a/release/installbuilder/logo-installer-icon.png b/release/installbuilder/logo-installer-icon.png Binary files differnew file mode 100644 index 00000000..c39541bb --- /dev/null +++ b/release/installbuilder/logo-installer-icon.png diff --git a/release/installbuilder/logo-installer.png b/release/installbuilder/logo-installer.png Binary files differnew file mode 100644 index 00000000..42de543d --- /dev/null +++ b/release/installbuilder/logo-installer.png diff --git a/release/installbuilder/logo.ico b/release/installbuilder/logo.ico Binary files differnew file mode 100644 index 00000000..8e08e8cc --- /dev/null +++ b/release/installbuilder/logo.ico diff --git a/release/installbuilder/mitmproxy.xml b/release/installbuilder/mitmproxy.xml new file mode 100644 index 00000000..e4ad00a3 --- /dev/null +++ b/release/installbuilder/mitmproxy.xml @@ -0,0 +1,135 @@ +<project>
+ <shortName>mitmproxy</shortName>
+ <fullName>mitmproxy</fullName>
+ <version>1.0</version>
+ <leftImage>logo-installer.png</leftImage>
+ <logoImage>logo-installer-icon.png</logoImage>
+ <componentList>
+ <component>
+ <name>default</name>
+ <description>Default Component</description>
+ <canBeEdited>1</canBeEdited>
+ <selected>1</selected>
+ <show>1</show>
+ <folderList>
+ <folder>
+ <description>Program Files</description>
+ <destination>${installdir}</destination>
+ <name>programfiles</name>
+ <platforms>all</platforms>
+ <distributionFileList>
+ <distributionFile>
+ <origin>logo.ico</origin>
+ </distributionFile>
+ </distributionFileList>
+ </folder>
+ <folder>
+ <description>Program Files/bin</description>
+ <destination>${installdir}/bin</destination>
+ <name>binaries</name>
+ <platforms>all</platforms>
+ <distributionFileList>
+ <distributionFile>
+ <allowWildcards>1</allowWildcards>
+ <excludeFiles>*/patho*</excludeFiles>
+ <origin>../build/binaries/${platform_name}/*</origin>
+ </distributionFile>
+ </distributionFileList>
+ </folder>
+ </folderList>
+ <postInstallationActionList>
+ <addDirectoryToPath>
+ <insertAt>end</insertAt>
+ <path>${installdir}/bin</path>
+ <scope>user</scope>
+ </addDirectoryToPath>
+ </postInstallationActionList>
+ <postUninstallationActionList>
+ <removeDirectoryFromPath>
+ <path>${installdir}/bin</path>
+ <scope>user</scope>
+ </removeDirectoryFromPath>
+ </postUninstallationActionList>
+ <startMenuShortcutList>
+ <startMenuShortcut>
+ <comment></comment>
+ <name>mitmproxy ui</name>
+ <runAsAdmin>0</runAsAdmin>
+ <runInTerminal>0</runInTerminal>
+ <windowsExec>${installdir}\bin\mitmweb.exe</windowsExec>
+ <windowsExecArgs></windowsExecArgs>
+ <windowsIcon>${installdir}/logo.ico</windowsIcon>
+ <windowsPath>${installdir}</windowsPath>
+ </startMenuShortcut>
+ <startMenuShortcut>
+ <comment></comment>
+ <name>mitmdump</name>
+ <runAsAdmin>0</runAsAdmin>
+ <runInTerminal>0</runInTerminal>
+ <windowsExec>${installdir}\bin\mitmdump.exe</windowsExec>
+ <windowsExecArgs></windowsExecArgs>
+ <windowsIcon>${installdir}/logo.ico</windowsIcon>
+ <windowsPath>${installdir}</windowsPath>
+ </startMenuShortcut>
+ </startMenuShortcutList>
+ </component>
+ </componentList>
+ <createOsxBundleDmg>1</createOsxBundleDmg>
+ <disableSplashScreen>1</disableSplashScreen>
+ <enableRollback>1</enableRollback>
+ <enableTimestamp>1</enableTimestamp>
+ <outputDirectory>../dist</outputDirectory>
+ <productDisplayIcon>logo.ico</productDisplayIcon>
+ <saveRelativePaths>1</saveRelativePaths>
+ <vendor>mitmproxy.org</vendor>
+ <windowsExecutableIcon>logo.ico</windowsExecutableIcon>
+ <finalPageActionList>
+ <runProgram>
+ <program>cmd</program>
+ <programArguments>/c start "mitmproxy ui" "${installdir}\bin\mitmweb.exe" &</programArguments>
+ <progressText>Launch mitmproxy ui now</progressText>
+ <ruleList>
+ <platformTest>
+ <type>windows</type>
+ </platformTest>
+ </ruleList>
+ </runProgram>
+ <runProgram>
+ <program>${installdir}/mitmproxy</program>
+ <programArguments>&</programArguments>
+ <progressText>Launch mitmproxy now</progressText>
+ <ruleList>
+ <platformTest>
+ <negate>1</negate>
+ <type>windows</type>
+ </platformTest>
+ </ruleList>
+ </runProgram>
+ </finalPageActionList>
+ <parameterList>
+ <directoryParameter>
+ <name>installdir</name>
+ <description>Installer.Parameter.installdir.description</description>
+ <explanation>Installer.Parameter.installdir.explanation</explanation>
+ <value></value>
+ <default>${platform_install_prefix}/${product_shortname}</default>
+ <allowEmptyValue>0</allowEmptyValue>
+ <ask>yes</ask>
+ <cliOptionName>prefix</cliOptionName>
+ <mustBeWritable>yes</mustBeWritable>
+ <mustExist>0</mustExist>
+ <width>40</width>
+ <postShowPageActionList>
+ <!-- This will skip the readytoinstall page -->
+ <setInstallerVariable name="next_page" value="installation"/>
+ </postShowPageActionList>
+ <preShowPageActionList>
+ <setInstallerVariable>
+ <name>ui.button(next).text</name>
+ <value>${msg(Installer.Button.Install)}</value>
+ </setInstallerVariable>
+ </preShowPageActionList>
+ </directoryParameter>
+ </parameterList>
+</project>
+
diff --git a/release/rtool.py b/release/rtool.py index 4a6d1e16..59899510 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -13,6 +13,7 @@ import zipfile from os.path import join, abspath, dirname, exists, basename import click +import cryptography.fernet import pysftp # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -37,6 +38,12 @@ else: def Archive(name): return tarfile.open(name, "w:gz") +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) + ROOT_DIR = abspath(join(dirname(__file__), "..")) RELEASE_DIR = join(ROOT_DIR, "release") @@ -47,7 +54,7 @@ PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") # PyInstaller 3.2 does not bundle pydivert's Windivert binaries PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") -PYINSTALLER_DIST = join(BUILD_DIR, "binaries") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) VENV_DIR = join(BUILD_DIR, "venv") @@ -91,11 +98,6 @@ def get_snapshot_version() -> str: def archive_name(bdist: str) -> str: - platform_tag = { - "Darwin": "osx", - "Windows": "win32", - "Linux": "linux" - }.get(platform.system(), platform.system()) if platform.system() == "Windows": ext = "zip" else: @@ -103,7 +105,7 @@ def archive_name(bdist: str) -> str: return "{project}-{version}-{platform}.{ext}".format( project=bdist, version=get_version(), - platform=platform_tag, + platform=PLATFORM_TAG, ext=ext ) @@ -114,6 +116,19 @@ def wheel_name() -> str: ) +def installer_name() -> str: + ext = { + "Windows": "exe", + "Darwin": "dmg", + "Linux": "run" + }[platform.system()] + return "mitmproxy-{version}-{platform}-installer.{ext}".format( + version=get_version(), + platform=PLATFORM_TAG, + ext=ext, + ) + + @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() @@ -130,6 +145,24 @@ def cli(): pass +@cli.command("encrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def encrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.encrypt(infile.read())) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) + + @cli.command("contributors") def contributors(): """ @@ -238,7 +271,8 @@ def upload_release(username, password, repository): @click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) @click.option("--wheel/--no-wheel", default=False) @click.option("--bdist/--no-bdist", default=False) -def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist): +@click.option("--installer/--no-installer", default=False) +def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer): """ Upload snapshot to snapshot server """ @@ -256,6 +290,8 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, if bdist: for bdist in sorted(BDISTS.keys()): files.append(archive_name(bdist)) + if installer: + files.append(installer_name()) for f in files: local_path = join(DIST_DIR, f) diff --git a/release/setup.py b/release/setup.py index 1d60e46c..01d0672d 100644 --- a/release/setup.py +++ b/release/setup.py @@ -8,6 +8,7 @@ setup( "click>=6.2, <7.0", "twine>=1.6.5, <1.9", "pysftp==0.2.8", + "cryptography>=1.6, <1.7", ], entry_points={ "console_scripts": [ @@ -2,7 +2,7 @@ max-line-length = 140 max-complexity = 25 ignore = E251,C901,W503 -exclude = mitmproxy/contrib/*,test/mitmproxy/data/* +exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/* addons = file,open,basestring,xrange,unicode,long,cmp [tool:pytest] @@ -1,8 +1,8 @@ -from setuptools import setup, find_packages -from codecs import open import os +import runpy +from codecs import open -from mitmproxy import version +from setuptools import setup, find_packages # Based on https://github.com/pypa/sampleproject/blob/master/setup.py # and https://python-packaging-user-guide.readthedocs.org/ @@ -12,9 +12,11 @@ here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() +VERSION = runpy.run_path(os.path.join(here, "mitmproxy", "version.py"))["VERSION"] + setup( name="mitmproxy", - version=version.VERSION, + version=VERSION, description="An interactive, SSL-capable, man-in-the-middle HTTP proxy for penetration testers and software developers.", long_description=long_description, url="http://mitmproxy.org", @@ -62,7 +64,7 @@ setup( "click>=6.2, <7", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "construct>=2.8, <2.9", - "cryptography>=1.3, <1.7", + "cryptography>=1.3, <1.8", "cssutils>=1.0.1, <1.1", "Flask>=0.10.1, <0.12", "h2>=2.5.0, <3", @@ -106,6 +108,7 @@ setup( "sphinx>=1.3.5, <1.6", "sphinx-autobuild>=0.5.2, <0.7", "sphinxcontrib-documentedlist>=0.4.0, <0.5", + "docutils==0.12", # temporary pin, https://github.com/chintal/sphinxcontrib-documentedlist/pull/3 "sphinx_rtd_theme>=0.1.9, <0.2", ], 'contentviews': [ diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 06463fa3..777f8f4d 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -18,13 +18,6 @@ import watchdog.events from .. import tutils as ttutils -def test_ns(): - n = script.NS({}) - n.one = "one" - assert n.one == "one" - assert n.__dict__["ns"]["one"] == "one" - - def test_scriptenv(): with taddons.context() as tctx: with script.scriptenv("path", []): diff --git a/test/mitmproxy/addons/test_stickyauth.py b/test/mitmproxy/addons/test_stickyauth.py index 490e9aac..df74f44d 100644 --- a/test/mitmproxy/addons/test_stickyauth.py +++ b/test/mitmproxy/addons/test_stickyauth.py @@ -15,7 +15,8 @@ def test_configure(): def test_simple(): r = stickyauth.StickyAuth() - with taddons.context(): + with taddons.context() as tctx: + tctx.configure(r, stickyauth=".*") f = tflow.tflow(resp=True) f.request.headers["authorization"] = "foo" r.request(f) diff --git a/test/mitmproxy/console/test_pathedit.py b/test/mitmproxy/console/test_pathedit.py index 40d55353..b326ed6d 100644 --- a/test/mitmproxy/console/test_pathedit.py +++ b/test/mitmproxy/console/test_pathedit.py @@ -54,7 +54,7 @@ class TestPathEdit: def test_keypress(self): - pe = pathedit.PathEdit() + pe = pathedit.PathEdit("", "") with patch('urwid.widget.Edit.get_edit_text') as get_text, \ patch('urwid.widget.Edit.set_edit_text') as set_text: diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 8db2507f..610c9dad 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -150,7 +150,7 @@ class TestHARDump: def test_format_cookies(self): m, sc = tscript("complex/har_dump.py", "-") - format_cookies = sc.ns.ns["format_cookies"] + format_cookies = sc.ns.format_cookies CA = cookies.CookieAttrs diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 85684df9..3ec74b20 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -26,6 +26,8 @@ def test_check_type(): typecheck.check_type("foo", 42, str) with pytest.raises(TypeError): typecheck.check_type("foo", None, str) + with pytest.raises(TypeError): + typecheck.check_type("foo", b"foo", str) def test_check_union(): @@ -44,5 +46,14 @@ def test_check_tuple(): typecheck.check_type("foo", (42, 42), typing.Tuple[int, str]) with pytest.raises(TypeError): typecheck.check_type("foo", ("42", 42), typing.Tuple[int, str]) - typecheck.check_type("foo", (42, "42"), typing.Tuple[int, str]) + + +def test_check_sequence(): + typecheck.check_type("foo", [10], typing.Sequence[int]) + with pytest.raises(TypeError): + typecheck.check_type("foo", ["foo"], typing.Sequence[int]) + with pytest.raises(TypeError): + typecheck.check_type("foo", [10, "foo"], typing.Sequence[int]) + with pytest.raises(TypeError): + typecheck.check_type("foo", [b"foo"], typing.Sequence[str]) @@ -8,7 +8,7 @@ basepython = python3.5 deps = {env:CI_DEPS:} -rrequirements.txt -passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* OPENSSL_* +passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* OPENSSL_* RTOOL_* setenv = HOME = {envtmpdir} commands = mitmdump --sysinfo @@ -22,7 +22,7 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] commands = mitmdump --sysinfo - flake8 --jobs 8 --count mitmproxy pathod examples test + flake8 --jobs 8 --count mitmproxy pathod examples test release rstcheck README.rst mypy --silent-imports \ mitmproxy/addons \ |
