From f95784ba9d473f77d18f5391ca45da4d4f1d6e50 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 15 Sep 2018 11:54:18 +0000 Subject: update readme slogan --- README.rst | 8 ++++---- docs/src/content/_index.md | 9 +++++---- release/docker/README.md | 2 +- setup.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index d7100374..29c542c6 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ mitmproxy This repository contains the **mitmproxy** and **pathod** projects. -``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console -interface. +``mitmproxy`` is an interactive, SSL/TLS-capable intercepting proxy with a console +interface for HTTP/1, HTTP/2, and WebSockets. ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP. @@ -28,7 +28,7 @@ and pathod websites. The documentation for mitmproxy is available on our website: -|mitmproxy_docs_stable| |mitmproxy_docs_master| +|mitmproxy_docs_stable| |mitmproxy_docs_master| Join our discussion forum on Discourse to ask questions, help @@ -153,7 +153,7 @@ with the following command: .. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/api/docs-stable-brightgreen.svg :target: https://docs.mitmproxy.org/stable/ :alt: mitmproxy documentation stable - + .. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/api/docs-master-brightgreen.svg :target: https://docs.mitmproxy.org/master/ :alt: mitmproxy documentation master diff --git a/docs/src/content/_index.md b/docs/src/content/_index.md index cd368df0..6283343d 100644 --- a/docs/src/content/_index.md +++ b/docs/src/content/_index.md @@ -11,8 +11,7 @@ menu: The mitmproxy project's tools are a set of front-ends that expose common underlying functionality. -**mitmproxy** is an interactive man-in-the-middle proxy for HTTP and HTTPS -with a console interface. +**mitmproxy** is an interactive, SSL/TLS-capable intercepting proxy with a console interface for HTTP/1, HTTP/2, and WebSockets. **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP. @@ -21,6 +20,9 @@ with a console interface. Documentation, tutorials and distribution packages can be found on the [mitmproxy website](https://mitmproxy.org). +Development information and our source code can be found in our +[GitHub repository](https://github.com/mitmproxy/mitmproxy). + ## Features @@ -29,8 +31,7 @@ Documentation, tutorials and distribution packages can be found on the - Replay the client-side of an HTTP conversations - Replay HTTP responses of a previously recorded server - Reverse proxy mode to forward traffic to a specified server -- Transparent proxy mode on OSX and Linux +- Transparent proxy mode on macOS and Linux - Make scripted changes to HTTP traffic using Python - SSL/TLS certificates for interception are generated on the fly - And much, much more... - diff --git a/release/docker/README.md b/release/docker/README.md index 4511a33a..b676d3ae 100644 --- a/release/docker/README.md +++ b/release/docker/README.md @@ -1,6 +1,6 @@ # mitmproxy -Containerized version of [mitmproxy](https://mitmproxy.org/), an interactive SSL-capable intercepting HTTP proxy. +Containerized version of [mitmproxy](https://mitmproxy.org/): an interactive, SSL/TLS-capable intercepting proxy for HTTP/1, HTTP/2, and WebSockets. # Usage diff --git a/setup.py b/setup.py index dd48f308..7ebeb208 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ with open(os.path.join(here, "mitmproxy", "version.py")) as f: setup( name="mitmproxy", version=VERSION, - description="An interactive, SSL-capable, man-in-the-middle HTTP proxy for penetration testers and software developers.", + description="An interactive, SSL/TLS-capable intercepting proxy for HTTP/1, HTTP/2, and WebSockets.", long_description=long_description, url="http://mitmproxy.org", author="Aldo Cortesi", -- cgit v1.2.3 From 4bd992d63c3fd37a921ff48f06382bab2003a4e8 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Mon, 17 Dec 2018 00:08:27 +0530 Subject: external-viewer-fix --- mitmproxy/tools/console/master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index dd15a2f5..6ab9ba5a 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -120,7 +120,7 @@ class ConsoleMaster(master.Master): with open(fd, "w" if text else "wb") as f: f.write(data) # if no EDITOR is set, assume 'vi' - c = os.environ.get("EDITOR") or "vi" + c = os.environ.get("MITMPROXY_EDITOR") or os.environ.get("EDITOR") or "vi" cmd = shlex.split(c) cmd.append(name) with self.uistopped(): @@ -159,7 +159,7 @@ class ConsoleMaster(master.Master): shell = True if not cmd: # hm which one should get priority? - c = os.environ.get("PAGER") or os.environ.get("EDITOR") + c = os.environ.get("MITMPROXY_EDITOR") or os.environ.get("PAGER") or os.environ.get("EDITOR") if not c: c = "less" cmd = shlex.split(c) -- cgit v1.2.3 From 407a5d71a86b715a3c4dce34e1918c35e9dcab79 Mon Sep 17 00:00:00 2001 From: BkPHcgQL3V Date: Wed, 19 Dec 2018 17:52:32 +0000 Subject: Tabular list flow --- mitmproxy/tools/console/common.py | 360 ++++++++++++++++++++++++++++-------- mitmproxy/tools/console/flowlist.py | 2 +- mitmproxy/tools/console/flowview.py | 2 +- mitmproxy/tools/console/palettes.py | 67 ++++++- 4 files changed, 356 insertions(+), 75 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 5d7ee09d..49f5c247 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,5 +1,8 @@ import platform import typing +import datetime +import time +import math from functools import lru_cache import urwid @@ -97,12 +100,171 @@ if urwid.util.detected_encoding and not IS_WSL: SYMBOL_MARK = u"\u25cf" SYMBOL_UP = u"\u21E7" SYMBOL_DOWN = u"\u21E9" + SYMBOL_ELLIPSIS = u"\u2026" else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" SYMBOL_MARK = "[m]" SYMBOL_UP = "^" SYMBOL_DOWN = " " + SYMBOL_ELLIPSIS = "~" + + +def fixlen(s, maxlen): + if len(s) <= maxlen: + return s.ljust(maxlen) + else: + return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS + + +def fixlen_r(s, maxlen): + if len(s) <= maxlen: + return s.rjust(maxlen) + else: + return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):] + + +class TruncatedText(urwid.Widget): + def __init__(self, text, attr, align='left'): + self.text = text + self.attr = attr + self.align = align + super(TruncatedText, self).__init__() + + def pack(self, size, focus=False): + return (len(self.text), 1) + + def rows(self, size, focus=False): + return 1 + + def render(self, size, focus=False): + text = self.text + attr = self.attr + if self.align == 'right': + text = text[::-1] + attr = attr[::-1] + + text_len = len(text) # TODO: unicode? + if size is not None and len(size) > 0: + width = size[0] + else: + width = text_len + + if width >= text_len: + remaining = width - text_len + if remaining > 0: + c_text = text + ' ' * remaining + c_attr = attr + [('text', remaining)] + else: + c_text = text + c_attr = attr + else: + visible_len = width - len(SYMBOL_ELLIPSIS) + visible_text = text[0:visible_len] + c_text = visible_text + SYMBOL_ELLIPSIS + c_attr = (urwid.util.rle_subseg(attr, 0, len(visible_text.encode())) + + [('focus', len(SYMBOL_ELLIPSIS.encode()))]) + + if self.align == 'right': + c_text = c_text[::-1] + c_attr = c_attr[::-1] + + return urwid.TextCanvas([c_text.encode()], [c_attr], maxcol=width) + + +def truncated_plain(text, attr, align='left'): + return TruncatedText(text, [(attr, len(text.encode()))], align) + + +# Work around https://github.com/urwid/urwid/pull/330 +def rle_append_beginning_modify(rle, a_r): + """ + Append (a, r) (unpacked from *a_r*) to BEGINNING of rle. + Merge with first run when possible + + MODIFIES rle parameter contents. Returns None. + """ + a, r = a_r + if not rle: + rle[:] = [(a, r)] + else: + al, run = rle[0] + if a == al: + rle[0] = (a, run + r) + else: + rle[0:0] = [(a, r)] + + +def colorize_host(s): + if len(s) == 0 or s[0] == '[' or s.split('.')[-1].isdigit(): + main_part = -1 + else: + main_part = 1 # TODO: second-level domains (https://publicsuffix.org/list/) + part = 0 + attr = [] + for i in reversed(range(len(s))): + c = s[i] + if c == '.': + part += 1 + if c in ".:[]": + a = 'url_punctuation' + elif part == main_part: + a = 'url_domain' + else: + a = 'text' + rle_append_beginning_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_req(s): + path = s.split('?', 2)[0] + i_query = len(path) + i_last_slash = path.rfind('/') + i_ext = path[i_last_slash + 1:].rfind('.') + i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s) + in_val = False + attr = [] + for i in range(len(s)): + c = s[i] + if ((i < i_query and c == '/') or + (i < i_query and i > i_last_slash and c == '.') or + (i == i_query)): + a = 'url_punctuation' + elif i > i_query: + if in_val: + if c == '&': + in_val = False + a = 'url_punctuation' + else: + a = 'url_query_value' + else: + if c == '=': + in_val = True + a = 'url_punctuation' + else: + a = 'url_query_key' + elif i > i_ext: + a = 'url_extension' + elif i > i_last_slash: + a = 'url_filename' + else: + a = 'text' + urwid.util.rle_append_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_url(url): + parts = url.split('/', 3) + if len(parts) < 4 or len(parts[1]) > 0 or parts[0][-1:] != ':': + return [('error', len(url))] # bad URL + schemes = { + 'http:': 'scheme_http', + 'https:': 'scheme_https', + } + return [ + (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), + ('url_punctuation', 3), # :// + ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) @lru_cache(maxsize=800) @@ -110,50 +272,71 @@ def raw_format_flow(f): f = dict(f) pile = [] req = [] - if f["extended"]: - req.append( - fcol( - human.format_timestamp(f["req_timestamp"]), - "highlight" - ) - ) - else: - req.append(fcol(">>" if f["focus"] else " ", "focus")) - if f["marked"]: - req.append(fcol(SYMBOL_MARK, "mark")) + cursor = [' ', 'focus'] + if f.get('resp_is_replay', False): + cursor[0] = SYMBOL_REPLAY + cursor[1] = 'replay' + if f['marked']: + if cursor[0] == ' ': + cursor[0] = SYMBOL_MARK + cursor[1] = 'mark' + if f['focus']: + cursor[0] = '>' - if f["req_is_replay"]: - req.append(fcol(SYMBOL_REPLAY, "replay")) + req.append(fcol(*cursor)) - req.append(fcol(f["req_method"], "method")) + if f["two_line"]: + req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left')) + pile.append(urwid.Columns(req, dividechars=1)) - preamble = sum(i[1] for i in req) + len(req) - 1 + req = [] + req.append(fcol(' ', 'text')) if f["intercepted"] and not f["acked"]: uc = "intercept" - elif "resp_code" in f or "err_msg" in f: - uc = "text" + elif "resp_code" in f or f["err_msg"] is not None: + uc = "highlight" else: uc = "title" - url = f["req_url"] - - if f["max_url_len"] and len(url) > f["max_url_len"]: - url = url[:f["max_url_len"]] + "…" + if f["extended"]: + s = human.format_timestamp(f["req_timestamp"]) + else: + s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S") + req.append(fcol(s, uc)) + + methods = { + 'GET': 'method_get', + 'POST': 'method_post', + } + uc = methods.get(f["req_method"], "method_other") + if f['extended']: + req.append(fcol(f["req_method"], uc)) + if f["req_promise"]: + req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + else: + if f["req_promise"]: + uc = 'method_http2_push' + req.append(("fixed", 4, truncated_plain(f["req_method"], uc))) - if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): - url += " " + f["req_http_version"] - req.append( - urwid.Text([(uc, url)]) - ) + if f["two_line"]: + req.append(fcol(f["req_http_version"], 'text')) + else: + schemes = { + 'http': 'scheme_http', + 'https': 'scheme_https', + } + req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other"))) - pile.append(urwid.Columns(req, dividechars=1)) + req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right'))) + req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left'))) - resp = [] - resp.append( - ("fixed", preamble, urwid.Text("")) - ) + ret = (' ' * len(SYMBOL_RETURN), 'text') + status = ('', 'text') + content = ('', 'text') + size = ('', 'text') + duration = ('', 'text') if "resp_code" in f: codes = { @@ -163,79 +346,112 @@ def raw_format_flow(f): 5: "code_500", } ccol = codes.get(f["resp_code"] // 100, "code_other") - resp.append(fcol(SYMBOL_RETURN, ccol)) - if f["resp_is_replay"]: - resp.append(fcol(SYMBOL_REPLAY, "replay")) - resp.append(fcol(f["resp_code"], ccol)) - if f["extended"]: - resp.append(fcol(f["resp_reason"], ccol)) - if f["intercepted"] and f["resp_code"] and not f["acked"]: - rc = "intercept" + ret = (SYMBOL_RETURN, ccol) + status = (str(f["resp_code"]), ccol) + + if f["resp_len"] < 0: + if f["intercepted"] and f["resp_code"] and not f["acked"]: + rc = "intercept" + else: + rc = "content_none" + + if f["resp_len"] == -1: + contentdesc = "[content missing]" + else: + contentdesc = "[no content]" + content = (contentdesc, rc) else: - rc = "text" - - if f["resp_ctype"]: - resp.append(fcol(f["resp_ctype"], rc)) - resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["roundtrip"], rc)) + if f["resp_ctype"]: + ctype = f["resp_ctype"].split(";")[0] + if ctype.endswith('/javascript'): + rc = 'content_script' + elif ctype.startswith('text/'): + rc = 'content_text' + elif (ctype.startswith('image/') or + ctype.startswith('video/') or + ctype.startswith('font/') or + "/x-font-" in ctype): + rc = 'content_media' + elif ctype.endswith('/json') or ctype.endswith('/xml'): + rc = 'content_data' + elif ctype.startswith('application/'): + rc = 'content_raw' + else: + rc = 'content_other' + content = (ctype, rc) + + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99)) + + size_str = human.pretty_size(f["resp_len"]) + if not f['extended']: + # shorten to 5 chars max + if len(size_str) > 5: + size_str = size_str[0:4].rstrip('.') + size_str[-1:] + size = (size_str, rc) + + if f['duration'] is not None: + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99)) + duration = (human.pretty_duration(f['duration']), rc) elif f["err_msg"]: - resp.append(fcol(SYMBOL_RETURN, "error")) - resp.append( - urwid.Text([ - ( - "error", - f["err_msg"] - ) - ]) - ) - pile.append(urwid.Columns(resp, dividechars=1)) + status = ('Err', 'error') + content = f["err_msg"], 'error' + + if f["two_line"]: + req.append(fcol(*ret)) + req.append(fcol(fixlen(status[0], 3), status[1])) + req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right'))) + if f['extended']: + req.append(fcol(*size)) + else: + req.append(fcol(fixlen_r(size[0], 5), size[1])) + req.append(fcol(fixlen_r(duration[0], 5), duration[1])) + + pile.append(urwid.Columns(req, dividechars=1, min_width=15)) + return urwid.Pile(pile) -def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): +def format_flow(f, focus, extended=False, hostheader=False, cols=False): acked = False if f.reply and f.reply.state == "committed": acked = True - pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in f.metadata else '' d = dict( focus=focus, extended=extended, - max_url_len=max_url_len, + two_line=extended or cols < 100, intercepted=f.intercepted, acked=acked, req_timestamp=f.request.timestamp_start, req_is_replay=f.request.is_replay, - req_method=f.request.method + pushed, + req_method=f.request.method, + req_promise='h2-pushed-stream' in f.metadata, req_url=f.request.pretty_url if hostheader else f.request.url, + req_scheme=f.request.scheme, + req_host=f.request.pretty_host if hostheader else f.request.host, + req_path=f.request.path, req_http_version=f.request.http_version, err_msg=f.error.msg if f.error else None, marked=f.marked, ) if f.response: if f.response.raw_content: - contentdesc = human.pretty_size(len(f.response.raw_content)) + content_len = len(f.response.raw_content) elif f.response.raw_content is None: - contentdesc = "[content missing]" + content_len = -1 else: - contentdesc = "[no content]" - duration = 0 + content_len = -2 + duration = None if f.response.timestamp_end and f.request.timestamp_start: duration = f.response.timestamp_end - f.request.timestamp_start - roundtrip = human.pretty_duration(duration) d.update(dict( resp_code=f.response.status_code, resp_reason=f.response.reason, resp_is_replay=f.response.is_replay, - resp_clen=contentdesc, - roundtrip=roundtrip, + resp_len=content_len, + resp_ctype=f.response.headers.get("content-type"), + duration=duration, )) - t = f.response.headers.get("content-type") - if t: - d["resp_ctype"] = t.split(";")[0] - else: - d["resp_ctype"] = "" - return raw_format_flow(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index e947a582..63e67327 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -18,7 +18,7 @@ class FlowItem(urwid.WidgetWrap): self.flow, self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, ) def selectable(self): diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index b4e3876f..5466319a 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -38,7 +38,7 @@ class FlowViewHeader(urwid.WidgetWrap): False, extended=True, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 7930c4a3..405f1a6c 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -22,7 +22,12 @@ class Palette: 'option_selected_key', # List and Connections - 'method', 'focus', + 'method', + 'method_get', 'method_post', 'method_other', 'method_http2_push', + 'scheme_http', 'scheme_https', 'scheme_other', + 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', + 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', + 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', @@ -36,6 +41,7 @@ class Palette: # Commander 'commander_command', 'commander_invalid', 'commander_hint' ] + _fields.extend(['gradient_%02d' % i for i in range(100)]) high: typing.Mapping[str, typing.Sequence[str]] = None def palette(self, transparent): @@ -68,6 +74,27 @@ class Palette: return l +def gen_gradient(palette, cols): + for i in range(100): + palette['gradient_%02d' % i] = (cols[i * len(cols) // 100], 'default') + + +def gen_rgb_gradient(palette, cols): + parts = len(cols) - 1 + for i in range(100): + p = i / 100 + idx = int(p * parts) + t0 = cols[idx] + t1 = cols[idx + 1] + pp = p * parts % 1 + t = ( + round(t0[0] + (t1[0] - t0[0]) * pp), + round(t0[1] + (t1[1] - t0[1]) * pp), + round(t0[2] + (t1[2] - t0[2]) * pp), + ) + palette['gradient_%02d' % i] = ("#%x%x%x" % t, 'default') + + class LowDark(Palette): """ @@ -95,6 +122,30 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark cyan', 'default'), + method_post = ('dark red', 'default'), + method_other = ('dark magenta', 'default'), + method_http2_push = ('dark gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('dark green', 'default'), + scheme_other = ('dark magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('white', 'default'), + url_filename = ('dark cyan', 'default'), + url_extension = ('light gray', 'default'), + url_query_key = ('white', 'default'), + url_query_value = ('light gray', 'default'), + + content_none = ('dark gray', 'default'), + content_text = ('light gray', 'default'), + content_script = ('dark green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('dark red', 'default'), + content_other = ('dark magenta', 'default'), + focus = ('yellow', 'default'), code_200 = ('dark green', 'default'), @@ -127,6 +178,7 @@ class LowDark(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('dark gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Dark(LowDark): @@ -312,8 +364,20 @@ class SolarizedDark(LowDark): # List and Connections method = (sol_cyan, 'default'), + method_http2_push = (sol_base01, 'default'), focus = (sol_base1, 'default'), + url_punctuation = ('h242', 'default'), + url_domain = ('h252', 'default'), + url_filename = ('h132', 'default'), + url_extension = ('h96', 'default'), + url_query_key = ('h37', 'default'), + url_query_value = ('h30', 'default'), + + content_none = (sol_base01, 'default'), + content_text = (sol_base1, 'default'), + content_media = (sol_blue, 'default'), + code_200 = (sol_green, 'default'), code_300 = (sol_blue, 'default'), code_400 = (sol_orange, 'default',), @@ -342,6 +406,7 @@ class SolarizedDark(LowDark): commander_invalid = (sol_orange, 'default'), commander_hint = (sol_base00, 'default'), ) + gen_rgb_gradient(high, [(15, 0, 0), (15, 15, 0), (0, 15, 0), (0, 15, 15), (0, 0, 15)]) DEFAULT = "dark" -- cgit v1.2.3 From cc33f40f29dcd250e2dc3df77412fdec91f2d4eb Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Sat, 29 Dec 2018 21:24:46 +0800 Subject: Fix a failing test on macOS Mojave --- test/mitmproxy/test_proxy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 00086c4b..c8cf6c33 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,4 +1,5 @@ import argparse +import platform from unittest import mock import pytest @@ -52,8 +53,11 @@ class TestProcessProxyOptions: class TestProxyServer: @skip_windows + @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'], + reason='Skipping due to macOS Mojave') def test_err(self): - # binding to 0.0.0.0:1 works without special permissions on Windows + # binding to 0.0.0.0:1 works without special permissions on Windows and + # macOS Mojave conf = ProxyConfig(options.Options(listen_port=1)) with pytest.raises(Exception, match="Error starting proxy server"): ProxyServer(conf) -- cgit v1.2.3 From d6445fa5bac87679ca6830f880524c414c6b81bf Mon Sep 17 00:00:00 2001 From: Peter Smythe Date: Thu, 3 Jan 2019 07:54:57 +0200 Subject: Fix spelling --- docs/src/content/concepts-protocols.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md index fc056545..c79274bf 100644 --- a/docs/src/content/concepts-protocols.md +++ b/docs/src/content/concepts-protocols.md @@ -36,7 +36,7 @@ mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the major browser vendors have implemented it. Some websites are still having problems with correct HTTP/2 support in their -webservers and can cause errors, dropped connectiones, or simply no response at +webservers and can cause errors, dropped connections, or simply no response at all. We are trying to be as tolerant and forgiving as possible with the types of data we send and receive, but [some](https://github.com/mitmproxy/mitmproxy/issues/1745) -- cgit v1.2.3 From 2223520403aaa5581c1a3b8d74c0cf9f90c2c8c3 Mon Sep 17 00:00:00 2001 From: Peter Smythe Date: Thu, 3 Jan 2019 08:48:09 +0200 Subject: Fix space --- docs/src/content/tute-highscores.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/tute-highscores.md b/docs/src/content/tute-highscores.md index f5cbd7bc..2d03076d 100644 --- a/docs/src/content/tute-highscores.md +++ b/docs/src/content/tute-highscores.md @@ -67,7 +67,7 @@ timestamp. Looks pretty simple to mess with. Lets edit the score submission. First, select it in mitmproxy, then press enter to view it. Make sure you're -viewing the request, not the response -you can use +viewing the request, not the response - you can use tab to flick between the two. Now press e for edit. You'll be prompted for the part of the request you want to change - press r -- cgit v1.2.3 From 1ee54f7d739afb34063712b71db2bf892d1dd663 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Mon, 7 Jan 2019 23:17:11 +0530 Subject: option added --- mitmproxy/tools/console/consoleaddons.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..ba14eeb6 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -113,6 +113,11 @@ class ConsoleAddon: "console_mouse", bool, True, "Console mouse interaction." ) + loader.add_option( + "console_external_viewer_default", str, "vi", + "External viewer for flow body. Set environment variable $MITMPROXY_EDITOR " + "to change default." + ) @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: -- cgit v1.2.3 From c331ba91fca6e5d82ab09fba417d5ee0d37504bc Mon Sep 17 00:00:00 2001 From: Jihyun Yu Date: Fri, 18 Jan 2019 15:59:58 +0900 Subject: openbsd: divert-to requires exact listen address divert-to does not work with '0.0.0.0' or similar listen address, so we need to specify listen address that we provided to `pf`. --- docs/src/content/howto-transparent.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index ae36f579..9be1e2f8 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -124,7 +124,7 @@ doas pfctl -e You probably want a command like this: {{< highlight bash >}} -mitmproxy --mode transparent --showhost +mitmproxy --mode transparent --listen-host 127.0.0.1 --showhost {{< / highlight >}} The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells -- cgit v1.2.3 From 63a5e1d3ae8158149441bf0a9e4ca15fa4d971e7 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 18 Jan 2019 19:47:45 +0100 Subject: remove sudo:false flag https://blog.travis-ci.com/2018-11-19-required-linux-infrastructure-migration --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dca567bf..63c9b444 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: python branches: -- cgit v1.2.3 From 8b9583f19d6d20a06996a897c2df95c3a3b12733 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 18 Jan 2019 19:52:24 +0100 Subject: remove travis workaround --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 63c9b444..20afc279 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,6 @@ matrix: - python: 3.7 env: TOXENV=py37 dist: xenial - sudo: true # required workaround for https://github.com/travis-ci/travis-ci/issues/9815 - language: node_js node_js: "node" before_install: -- cgit v1.2.3 From 995d9f0f4946aacba8ad4ff6b26a56cfe705af66 Mon Sep 17 00:00:00 2001 From: Mumen Yassin Date: Wed, 23 Jan 2019 14:34:55 +0300 Subject: fixes copying on macos --- web/src/js/ducks/ui/keyboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/js/ducks/ui/keyboard.js b/web/src/js/ducks/ui/keyboard.js index ed4dbba5..007d24db 100644 --- a/web/src/js/ducks/ui/keyboard.js +++ b/web/src/js/ducks/ui/keyboard.js @@ -6,7 +6,7 @@ import * as modalActions from "./modal" export function onKeyDown(e) { //console.debug("onKeyDown", e) - if (e.ctrlKey) { + if (e.ctrlKey || e.metaKey) { return () => { } } -- cgit v1.2.3 From 106948d996d74bf5ff7e3511f35eefea0a90561f Mon Sep 17 00:00:00 2001 From: rpigott Date: Sun, 27 Jan 2019 00:59:26 -0800 Subject: update to wsproto 0.13 --- mitmproxy/proxy/protocol/websocket.py | 87 +++++++++++++++++------------------ setup.py | 2 +- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 0d1964a6..591bae7e 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -4,8 +4,9 @@ from OpenSSL import SSL import wsproto -from wsproto import events -from wsproto.connection import ConnectionType, WSConnection +from wsproto import events, WSConnection +from wsproto.connection import ConnectionType +from wsproto.events import AcceptConnection, CloseConnection, Message, Ping, Request from wsproto.extensions import PerMessageDeflate from mitmproxy import exceptions @@ -56,47 +57,44 @@ class WebSocketLayer(base.Layer): if 'Sec-WebSocket-Extensions' in handshake_flow.response.headers: if PerMessageDeflate.name in handshake_flow.response.headers['Sec-WebSocket-Extensions']: extensions = [PerMessageDeflate()] - self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER, - extensions=extensions) - self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT, - host=handshake_flow.request.host, - resource=handshake_flow.request.path, - extensions=extensions) + self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER) + self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT) + if extensions: - for conn in self.connections.values(): - conn.extensions[0].finalize(conn, handshake_flow.response.headers['Sec-WebSocket-Extensions']) + extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) - data = self.connections[self.server_conn].bytes_to_send() - self.connections[self.client_conn].receive_bytes(data) + request = Request(extensions = extensions, host = handshake_flow.request.host, target = handshake_flow.request.path) + data = self.connections[self.server_conn].send(request) + self.connections[self.client_conn].receive_data(data) event = next(self.connections[self.client_conn].events()) - assert isinstance(event, events.ConnectionRequested) + assert isinstance(event, events.Request) - self.connections[self.client_conn].accept(event) - self.connections[self.server_conn].receive_bytes(self.connections[self.client_conn].bytes_to_send()) - assert isinstance(next(self.connections[self.server_conn].events()), events.ConnectionEstablished) + data = self.connections[self.client_conn].send(AcceptConnection(extensions=extensions)) + self.connections[self.server_conn].receive_data(data) + assert isinstance(next(self.connections[self.server_conn].events()), events.AcceptConnection) def _handle_event(self, event, source_conn, other_conn, is_server): - if isinstance(event, events.DataReceived): - return self._handle_data_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PingReceived): - return self._handle_ping_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PongReceived): - return self._handle_pong_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.ConnectionClosed): - return self._handle_connection_closed(event, source_conn, other_conn, is_server) + if isinstance(event, events.Message): + return self._handle_message(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Ping): + return self._handle_ping(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Pong): + return self._handle_pong(event, source_conn, other_conn, is_server) + elif isinstance(event, events.CloseConnection): + return self._handle_close_connection(event, source_conn, other_conn, is_server) # fail-safe for unhandled events return True # pragma: no cover - def _handle_data_received(self, event, source_conn, other_conn, is_server): + def _handle_message(self, event, source_conn, other_conn, is_server): fb = self.server_frame_buffer if is_server else self.client_frame_buffer fb.append(event.data) if event.message_finished: original_chunk_sizes = [len(f) for f in fb] - if isinstance(event, events.TextReceived): + if isinstance(event, events.TextMessage): message_type = wsproto.frame_protocol.Opcode.TEXT payload = ''.join(fb) else: @@ -127,19 +125,20 @@ class WebSocketLayer(base.Layer): yield (payload[i:i + chunk_size], True if i + chunk_size >= len(payload) else False) for chunk, final in get_chunk(websocket_message.content): - self.connections[other_conn].send_data(chunk, final) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data = chunk, message_finished = final)) + other_conn.send(data) if self.flow.stream: - self.connections[other_conn].send_data(event.data, event.message_finished) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data = event.data, message_finished = event.message_finished)) + other_conn.send(data) return True - def _handle_ping_received(self, event, source_conn, other_conn, is_server): - # PING is automatically answered with a PONG by wsproto - self.connections[other_conn].ping() - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + def _handle_ping(self, event, source_conn, other_conn, is_server): + # Use event.response to create the approprate Pong response + data = self.connections[other_conn].send(Ping()) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) self.log( "Ping Received from {}".format("server" if is_server else "client"), "info", @@ -147,7 +146,7 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_pong_received(self, event, source_conn, other_conn, is_server): + def _handle_pong(self, event, source_conn, other_conn, is_server): self.log( "Pong Received from {}".format("server" if is_server else "client"), "info", @@ -155,14 +154,15 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_connection_closed(self, event, source_conn, other_conn, is_server): + def _handle_close_connection(self, event, source_conn, other_conn, is_server): self.flow.close_sender = "server" if is_server else "client" self.flow.close_code = event.code self.flow.close_reason = event.reason - self.connections[other_conn].close(event.code, event.reason) - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[other_conn].send(CloseConnection(code=event.code, reason=event.reason)) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) return False @@ -170,8 +170,7 @@ class WebSocketLayer(base.Layer): while True: try: payload = message_queue.get_nowait() - self.connections[endpoint].send_data(payload, final=True) - data = self.connections[endpoint].bytes_to_send() + data = self.connections[endpoint].send(Message(data = payload, message_finished = True)) endpoint.send(data) except queue.Empty: break @@ -197,8 +196,8 @@ class WebSocketLayer(base.Layer): is_server = (source_conn == self.server_conn) frame = websockets.Frame.from_file(source_conn.rfile) - self.connections[source_conn].receive_bytes(bytes(frame)) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[source_conn].receive_data(bytes(frame)) + source_conn.send(data) if close_received: return diff --git a/setup.py b/setup.py index 5833ce38..4c17bdc6 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ setup( "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", - "wsproto>=0.12.0,<0.13.0", + "wsproto>=0.13.0,<0.14.0", ], extras_require={ ':sys_platform == "win32"': [ -- cgit v1.2.3 From 2af71a4486494b64431558da2e068dc1babe65fa Mon Sep 17 00:00:00 2001 From: rpigott Date: Wed, 30 Jan 2019 18:12:05 -0800 Subject: Separate client and server PerMessageDefalte extension per #3460 --- mitmproxy/proxy/protocol/websocket.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 591bae7e..fd2aed15 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -53,24 +53,28 @@ class WebSocketLayer(base.Layer): self.connections: dict[object, WSConnection] = {} - extensions = [] + client_extensions = [] + server_extensions = [] if 'Sec-WebSocket-Extensions' in handshake_flow.response.headers: if PerMessageDeflate.name in handshake_flow.response.headers['Sec-WebSocket-Extensions']: - extensions = [PerMessageDeflate()] + client_extensions = [PerMessageDeflate()] + server_extensions = [PerMessageDeflate()] self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER) self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT) - if extensions: - extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + if client_extensions: + client_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + if server_extensions: + server_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) - request = Request(extensions = extensions, host = handshake_flow.request.host, target = handshake_flow.request.path) + request = Request(extensions = client_extensions, host = handshake_flow.request.host, target = handshake_flow.request.path) data = self.connections[self.server_conn].send(request) self.connections[self.client_conn].receive_data(data) event = next(self.connections[self.client_conn].events()) assert isinstance(event, events.Request) - data = self.connections[self.client_conn].send(AcceptConnection(extensions=extensions)) + data = self.connections[self.client_conn].send(AcceptConnection(extensions=server_extensions)) self.connections[self.server_conn].receive_data(data) assert isinstance(next(self.connections[self.server_conn].events()), events.AcceptConnection) -- cgit v1.2.3 From 92fc87bc4c3922758fe9862fd309886f06fcb16b Mon Sep 17 00:00:00 2001 From: rpigott Date: Wed, 30 Jan 2019 23:44:35 -0800 Subject: whitespace error --- mitmproxy/proxy/protocol/websocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index fd2aed15..191da7a2 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -64,7 +64,7 @@ class WebSocketLayer(base.Layer): if client_extensions: client_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) - if server_extensions: + if server_extensions: server_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) request = Request(extensions = client_extensions, host = handshake_flow.request.host, target = handshake_flow.request.path) -- cgit v1.2.3 From 7fa209790e0df869d0d3c96ba75cb1a85943fc7c Mon Sep 17 00:00:00 2001 From: rpigott Date: Thu, 31 Jan 2019 12:34:00 -0800 Subject: removed whitespace in kwargs --- mitmproxy/proxy/protocol/websocket.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 191da7a2..f5ac6a29 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -67,7 +67,7 @@ class WebSocketLayer(base.Layer): if server_extensions: server_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) - request = Request(extensions = client_extensions, host = handshake_flow.request.host, target = handshake_flow.request.path) + request = Request(extensions=client_extensions, host=handshake_flow.request.host, target=handshake_flow.request.path) data = self.connections[self.server_conn].send(request) self.connections[self.client_conn].receive_data(data) @@ -129,11 +129,11 @@ class WebSocketLayer(base.Layer): yield (payload[i:i + chunk_size], True if i + chunk_size >= len(payload) else False) for chunk, final in get_chunk(websocket_message.content): - data = self.connections[other_conn].send(Message(data = chunk, message_finished = final)) + data = self.connections[other_conn].send(Message(data=chunk, message_finished=final)) other_conn.send(data) if self.flow.stream: - data = self.connections[other_conn].send(Message(data = event.data, message_finished = event.message_finished)) + data = self.connections[other_conn].send(Message(data=event.data, message_finished=event.message_finished)) other_conn.send(data) return True @@ -174,7 +174,7 @@ class WebSocketLayer(base.Layer): while True: try: payload = message_queue.get_nowait() - data = self.connections[endpoint].send(Message(data = payload, message_finished = True)) + data = self.connections[endpoint].send(Message(data=payload, message_finished=True)) endpoint.send(data) except queue.Empty: break -- cgit v1.2.3 From cec8c67465206caa5b4040c5f34050c9ef06b687 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Sun, 3 Feb 2019 00:49:53 +0530 Subject: non ascii fix and tests --- test/mitmproxy/net/http/test_url.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/mitmproxy/net/http/test_url.py b/test/mitmproxy/net/http/test_url.py index ecf8e896..48277859 100644 --- a/test/mitmproxy/net/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -49,6 +49,17 @@ def test_parse(): url.parse('http://lo[calhost') +def test_ascii_check(): + + test_url = "https://xyz.tax-edu.net?flag=selectCourse&lc_id=42825&lc_name=茅莽莽猫氓猫氓".encode() + scheme, host, port, full_path = url.parse(test_url) + assert scheme == b'https' + assert host == b'xyz.tax-edu.net' + assert port == 443 + assert full_path == b'/?flag%3DselectCourse%26lc_id%3D42825%26lc_name%3D%E8%8C%85%E8%8E%BD%E8%8E' \ + b'%BD%E7%8C%AB%E6%B0%93%E7%8C%AB%E6%B0%93' + + @pytest.mark.skipif(sys.version_info < (3, 6), reason='requires Python 3.6 or higher') def test_parse_port_range(): # Port out of range @@ -61,6 +72,7 @@ def test_unparse(): assert url.unparse("http", "foo.com", 80, "/bar") == "http://foo.com/bar" assert url.unparse("https", "foo.com", 80, "") == "https://foo.com:80" assert url.unparse("https", "foo.com", 443, "") == "https://foo.com" + assert url.unparse("https", "foo.com", 443, "*") == "https://foo.com" # We ignore the byte 126: '~' because of an incompatibility in Python 3.6 and 3.7 @@ -131,3 +143,7 @@ def test_unquote(): assert url.unquote("foo") == "foo" assert url.unquote("foo%20bar") == "foo bar" assert url.unquote(surrogates_quoted) == surrogates + + +def test_hostport(): + assert url.hostport(b"https", b"foo.com", 8080) == b"foo.com:8080" -- cgit v1.2.3 From ba054b15f367d59139ec78fe03dc1c7d8fb099b5 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Sun, 3 Feb 2019 00:53:05 +0530 Subject: url-fix --- mitmproxy/net/http/url.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index f938cb12..d8e14aeb 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -21,16 +21,25 @@ def parse(url): Raises: ValueError, if the URL is not properly formatted. """ - parsed = urllib.parse.urlparse(url) + # Size of Ascii character after encoding is 1 byte which is same as its size + # But non-Ascii character's size after encoding will be more than its size + def ascii_check(l): + if len(l) == len(str(l).encode()): + return True + return False + + if isinstance(url, bytes): + url = url.decode() + if not ascii_check(url): + url = urllib.parse.urlsplit(url) + url = list(url) + url[3] = urllib.parse.quote(url[3]) + url = urllib.parse.urlunsplit(url) + parsed = urllib.parse.urlparse(url) if not parsed.hostname: raise ValueError("No hostname given") - if isinstance(url, bytes): - host = parsed.hostname - - # this should not raise a ValueError, - # but we try to be very forgiving here and accept just everything. else: host = parsed.hostname.encode("idna") if isinstance(parsed, urllib.parse.ParseResult): -- cgit v1.2.3 From 0d108fac86bec9fdb64c2a9f74bfb4f12f5b6aca Mon Sep 17 00:00:00 2001 From: ls Date: Sun, 17 Feb 2019 23:26:20 +0100 Subject: Fixed the exif .ksy files in the kaitai repo. Compiled the .ksy to python files and updated them in the contrib folder. --- mitmproxy/contrib/kaitaistruct/exif_be.py | 20 ++++++++++++++------ mitmproxy/contrib/kaitaistruct/exif_le.py | 20 ++++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py index 8a6e7a2b..88ce4e54 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_be.py +++ b/mitmproxy/contrib/kaitaistruct/exif_be.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2be() self.ifd0_ofs = self._io.read_u4be() @@ -25,6 +24,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2be() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifBe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2be()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2be()) self.length = self._io.read_u4be() @@ -552,7 +560,7 @@ class ExifBe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py index 84e53a38..e25a2fc9 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_le.py +++ b/mitmproxy/contrib/kaitaistruct/exif_le.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2le() self.ifd0_ofs = self._io.read_u4le() @@ -25,6 +24,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2le() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifLe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2le()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2le()) self.length = self._io.read_u4le() @@ -552,7 +560,7 @@ class ExifLe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) -- cgit v1.2.3 From 6e153b2c017be294a23e78469367346d0f9250e2 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Sun, 24 Feb 2019 01:45:45 +0530 Subject: filter unicode fix --- mitmproxy/flowfilter.py | 23 +++++++++++++++++++++++ test/mitmproxy/test_flowfilter.py | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 7f8df96f..0d8f1062 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -475,7 +475,30 @@ def _make(): parts.append(f) simplerex = "".join(c for c in pp.printables if c not in "()~'\"") + alphdevanagari = pp.pyparsing_unicode.Devanagari.alphas + alphcyrillic = pp.pyparsing_unicode.Cyrillic.alphas + alphgreek = pp.pyparsing_unicode.Greek.alphas + alphchinese = pp.pyparsing_unicode.Chinese.alphas + alpharabic = pp.pyparsing_unicode.Arabic.alphas + alphhebrew = pp.pyparsing_unicode.Hebrew.alphas + alphjapanese = pp.pyparsing_unicode.Japanese.alphas + alphkorean = pp.pyparsing_unicode.Korean.alphas + alphlatin1 = pp.pyparsing_unicode.Latin1.alphas + alphlatinA = pp.pyparsing_unicode.LatinA.alphas + alphlatinB = pp.pyparsing_unicode.LatinB.alphas + rex = pp.Word(simplerex) |\ + pp.Word(alphcyrillic) |\ + pp.Word(alphgreek) |\ + pp.Word(alphchinese) |\ + pp.Word(alpharabic) |\ + pp.Word(alphdevanagari) |\ + pp.Word(alphhebrew) |\ + pp.Word(alphjapanese) |\ + pp.Word(alphkorean) |\ + pp.Word(alphlatin1) |\ + pp.Word(alphlatinA) |\ + pp.Word(alphlatinB) |\ pp.QuotedString("\"", escChar='\\') |\ pp.QuotedString("'", escChar='\\') for klass in filter_rex: diff --git a/test/mitmproxy/test_flowfilter.py b/test/mitmproxy/test_flowfilter.py index 4eb37d81..d53cec7d 100644 --- a/test/mitmproxy/test_flowfilter.py +++ b/test/mitmproxy/test_flowfilter.py @@ -28,6 +28,9 @@ class TestParsing: self._dump(p) assert len(p.lst) == 2 + def test_non_ascii(self): + assert flowfilter.parse("~s шгн") + def test_naked_url(self): a = flowfilter.parse("foobar ~h rex") assert a.lst[0].expr == "foobar" @@ -173,10 +176,30 @@ class TestMatchingHTTPFlow: assert not self.q("~bq message", q) assert not self.q("~bq message", s) + s.response.text = 'яч' # Cyrillic + assert self.q("~bs яч", s) + s.response.text = '测试' # Chinese + assert self.q('~bs 测试', s) + s.response.text = 'ॐ' # Hindi + assert self.q('~bs ॐ', s) + s.response.text = 'لله' # Arabic + assert self.q('~bs لله', s) + s.response.text = 'θεός' # Greek + assert self.q('~bs θεός', s) + s.response.text = 'לוהים' # Hebrew + assert self.q('~bs לוהים', s) + s.response.text = '神' # Japanese + assert self.q('~bs 神', s) + s.response.text = '하나님' # Korean + assert self.q('~bs 하나님', s) + s.response.text = 'Äÿ' # Latin + assert self.q('~bs Äÿ', s) + assert not self.q("~bs nomatch", s) assert not self.q("~bs content", q) assert not self.q("~bs content", s) assert not self.q("~bs message", q) + s.response.text = 'message' assert self.q("~bs message", s) def test_body(self): -- cgit v1.2.3 From 9b7c2566d3e00e0bc4e4d2a21bb8d63a2b28d043 Mon Sep 17 00:00:00 2001 From: Rajat Gupta <35985127+rjt-gupta@users.noreply.github.com> Date: Thu, 28 Feb 2019 23:59:50 +0530 Subject: Update README.rst --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index d7100374..2804b1db 100644 --- a/README.rst +++ b/README.rst @@ -107,6 +107,12 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test cd test/mitmproxy/addons pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py + +To generate terminal report along with missing line numbers of individual test files or folders: + +.. code-block:: bash + + pytest --cov-report term-missing --cov mitmproxy.flowfilter --looponfail test_flowfilter.py As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes. -- cgit v1.2.3 From fad326a85f524f0b549dea87bceaf6dcb9e8e597 Mon Sep 17 00:00:00 2001 From: Rob Wills Date: Fri, 29 Mar 2019 15:11:17 -0700 Subject: Fix markdown typo on addons-scripting.md --- docs/src/content/addons-scripting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md index 4e9916ca..6a18eaf4 100644 --- a/docs/src/content/addons-scripting.md +++ b/docs/src/content/addons-scripting.md @@ -27,6 +27,6 @@ You can look at the [http][] module, or the [Request][], and [Response][] classes for other attributes that you can use when scripting. -[http][]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py +[http]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py [Request]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/request.py [Response]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/response.py -- cgit v1.2.3 From 6eb47a5fa5f848dd8356d0c64f3165c3e73fe333 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 2 Apr 2019 16:29:14 +0200 Subject: simplify testing instructions --- README.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 2804b1db..bccc6c5b 100644 --- a/README.rst +++ b/README.rst @@ -106,13 +106,7 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test .. code-block:: bash cd test/mitmproxy/addons - pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py - -To generate terminal report along with missing line numbers of individual test files or folders: - -.. code-block:: bash - - pytest --cov-report term-missing --cov mitmproxy.flowfilter --looponfail test_flowfilter.py + pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes. -- cgit v1.2.3 From 13ad81eaff35eef8e24a345bb665caf4fab77e43 Mon Sep 17 00:00:00 2001 From: Pierre Gordon Date: Tue, 9 Apr 2019 15:23:07 -0500 Subject: Remove the confdir argument Closes #3266 --- mitmproxy/tools/_main.py | 3 +-- mitmproxy/tools/cmdline.py | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index f1c763b2..b95d73ab 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -87,7 +87,7 @@ def run( arg_check.check() sys.exit(1) try: - opts.confdir = args.confdir + opts.set(*args.setoptions, defer=True) optmanager.load_paths( opts, os.path.join(opts.confdir, OPTIONS_FILE_NAME), @@ -110,7 +110,6 @@ def run( if args.commands: master.commands.dump() sys.exit(0) - opts.set(*args.setoptions, defer=True) if extra: opts.update(**extra(args)) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ad934ca2..a7820f76 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -20,12 +20,6 @@ def common_options(parser, opts): action='store_true', help="Show all commands and their signatures", ) - parser.add_argument( - "--confdir", - type=str, dest="confdir", default=core.CONF_DIR, - metavar="PATH", - help="Path to the mitmproxy config directory" - ) parser.add_argument( "--set", type=str, dest="setoptions", default=[], -- cgit v1.2.3 From bcd1bb23f6d56367d0f22c035b85078467a17ca1 Mon Sep 17 00:00:00 2001 From: Anthony Biondo Date: Tue, 16 Apr 2019 21:53:31 -0400 Subject: Fix #3502 by using email.utils.formatdate's usegmt argument. --- mitmproxy/net/http/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 48527d63..9491fc03 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -186,7 +186,7 @@ class Response(message.Message): d = parsedate_tz(self.headers[i]) if d: new = mktime_tz(d) + delta - self.headers[i] = formatdate(new) + self.headers[i] = formatdate(new, usegmt=True) c = [] for set_cookie_header in self.headers.get_all("set-cookie"): try: -- cgit v1.2.3 From a8489466c1b10af6a7510759362b9b930e6852f0 Mon Sep 17 00:00:00 2001 From: Anthony Biondo Date: Tue, 16 Apr 2019 22:11:27 -0400 Subject: update formatdate for cookies and tests to use GMT formatting --- mitmproxy/net/http/cookies.py | 2 +- test/mitmproxy/net/http/test_response.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 1472ab55..2745701f 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -304,7 +304,7 @@ def refresh_set_cookie_header(c: str, delta: int) -> str: e = email.utils.parsedate_tz(attrs["expires"]) if e: f = email.utils.mktime_tz(e) + delta - attrs.set_all("expires", [email.utils.formatdate(f)]) + attrs.set_all("expires", [email.utils.formatdate(f, usegmt=True)]) else: # This can happen when the expires tag is invalid. # reddit.com sends a an expires tag like this: "Thu, 31 Dec diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index f3470384..27c16be6 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -148,7 +148,7 @@ class TestResponseUtils: def test_refresh(self): r = tresp() n = time.time() - r.headers["date"] = email.utils.formatdate(n) + r.headers["date"] = email.utils.formatdate(n, usegmt=True) pre = r.headers["date"] r.refresh(946681202) assert pre == r.headers["date"] -- cgit v1.2.3 From 8d0c800d15039ff0a880051f4271acf668904e12 Mon Sep 17 00:00:00 2001 From: Pierre Gordon Date: Fri, 19 Apr 2019 12:39:53 -0500 Subject: Add --allow_hosts option Closes #3295 --- mitmproxy/options.py | 4 +++ mitmproxy/proxy/config.py | 25 +++++++++++++----- mitmproxy/proxy/root_context.py | 12 ++++----- mitmproxy/tools/cmdline.py | 1 + mitmproxy/tools/console/statusbar.py | 4 +++ test/mitmproxy/proxy/test_server.py | 51 ++++++++++++++++++++++++++++++++++-- 6 files changed, 82 insertions(+), 15 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index a6ab3d50..9fc6966b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -67,6 +67,10 @@ class Options(optmanager.OptManager): regular expression and matched on the ip or the hostname. """ ) + self.add_option( + "allow_hosts", Sequence[str], [], + "Opposite of --ignore_hosts." + ) self.add_option( "listen_host", str, "", "Address to bind proxy to." diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index f32d3086..55969d5e 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -14,7 +14,8 @@ CONF_BASENAME = "mitmproxy" class HostMatcher: - def __init__(self, patterns=tuple()): + def __init__(self, handle, patterns=tuple()): + self.handle = handle self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] @@ -22,8 +23,10 @@ class HostMatcher: if not address: return False host = "%s:%s" % address - if any(rex.search(host) for rex in self.regexes): - return True + if self.handle in ["ignore", "tcp"]: + return any(rex.search(host) for rex in self.regexes) + elif self.handle == "allow": + return any(not rex.search(host) for rex in self.regexes) else: return False @@ -36,7 +39,7 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_ignore: HostMatcher = None + self.check_filter: HostMatcher = None self.check_tcp: HostMatcher = None self.certstore: certs.CertStore = None self.upstream_server: typing.Optional[server_spec.ServerSpec] = None @@ -44,10 +47,18 @@ class ProxyConfig: options.changed.connect(self.configure) def configure(self, options: moptions.Options, updated: typing.Any) -> None: - if "ignore_hosts" in updated: - self.check_ignore = HostMatcher(options.ignore_hosts) + if options.allow_hosts and options.ignore_hosts: + raise exceptions.OptionsError("--ignore-hosts and --allow-hosts are mutually " + "exclusive; please choose one.") + + if options.ignore_hosts: + self.check_filter = HostMatcher("ignore", options.ignore_hosts) + elif options.allow_hosts: + self.check_filter = HostMatcher("allow", options.allow_hosts) + else: + self.check_filter = HostMatcher(False) if "tcp_hosts" in updated: - self.check_tcp = HostMatcher(options.tcp_hosts) + self.check_tcp = HostMatcher("tcp", options.tcp_hosts) certstore_path = os.path.expanduser(options.confdir) if not os.path.exists(os.path.dirname(certstore_path)): diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index eb0008cf..4805f874 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -48,17 +48,17 @@ class RootContext: raise exceptions.ProtocolException(str(e)) client_tls = tls.is_tls_record_magic(d) - # 1. check for --ignore - if self.config.check_ignore: - ignore = self.config.check_ignore(top_layer.server_conn.address) - if not ignore and client_tls: + # 1. check for filter + if self.config.check_filter: + is_filtered = self.config.check_filter(top_layer.server_conn.address) + if not is_filtered and client_tls: try: client_hello = tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: - ignore = self.config.check_ignore((client_hello.sni, 443)) - if ignore: + is_filtered = self.config.check_filter((client_hello.sni, 443)) + if is_filtered: return protocol.RawTCPLayer(top_layer, ignore=True) # 2. Always insert a TLS layer, even if there's neither client nor server tls. diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ad934ca2..e75134ac 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -65,6 +65,7 @@ def common_options(parser, opts): opts.make_parser(group, "listen_port", metavar="PORT", short="p") opts.make_parser(group, "server", short="n") opts.make_parser(group, "ignore_hosts", metavar="HOST") + opts.make_parser(group, "allow_hosts", metavar="HOST") opts.make_parser(group, "tcp_hosts", metavar="HOST") opts.make_parser(group, "upstream_auth", metavar="USER:PASS") opts.make_parser(group, "proxyauth", metavar="SPEC") diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 2d32f487..56f0674f 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -215,6 +215,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "I")) r.append("gnore:%d]" % len(self.master.options.ignore_hosts)) + elif self.master.options.allow_hosts: + r.append("[") + r.append(("heading_key", "A")) + r.append("llow:%d]" % len(self.master.options.allow_hosts)) if self.master.options.tcp_hosts: r.append("[") r.append(("heading_key", "T")) diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 01ab068d..b5852d60 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -78,6 +78,16 @@ class TcpMixin: self.options.ignore_hosts = self._ignore_backup del self._ignore_backup + def _allow_on(self): + assert not hasattr(self, "_allow_backup") + self._allow_backup = self.options.allow_hosts + self.options.allow_hosts = ["(127.0.0.1|None):\\d+"] + self.options.allow_hosts + + def _allow_off(self): + assert hasattr(self, "_allow_backup") + self.options.allow_hosts = self._allow_backup + del self._allow_backup + def test_ignore(self): n = self.pathod("304") self._ignore_on() @@ -111,6 +121,40 @@ class TcpMixin: self._ignore_off() + def test_allow(self): + n = self.pathod("304") + self._allow_on() + i = self.pathod("305") + i2 = self.pathod("306") + self._allow_off() + + assert n.status_code == 304 + assert i.status_code == 305 + assert i2.status_code == 306 + + assert any(f.response.status_code == 304 for f in self.master.state.flows) + assert any(f.response.status_code == 305 for f in self.master.state.flows) + assert any(f.response.status_code == 306 for f in self.master.state.flows) + + # Test that we get the original SSL cert + if self.ssl: + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) + + assert i_cert == i2_cert + assert i_cert != n_cert + + # Test Non-HTTP traffic + spec = "200:i0,@100:d0" # this results in just 100 random bytes + # mitmproxy responds with bad gateway + assert self.pathod(spec).status_code == 502 + self._allow_on() + + self.pathod(spec) # pathoc parses answer as HTTP + + self._allow_off() + def _tcpproxy_on(self): assert not hasattr(self, "_tcpproxy_backup") self._tcpproxy_backup = self.options.tcp_hosts @@ -852,10 +896,12 @@ class TestUpstreamProxySSL( def _host_pattern_on(self, attr): """ - Updates config.check_tcp or check_ignore, depending on attr. + Updates config.check_tcp or check_filter, depending on attr. """ assert not hasattr(self, "_ignore_%s_backup" % attr) backup = [] + handle = attr + attr = "filter" if attr in ["allow", "ignore"] else attr for proxy in self.chain: old_matcher = getattr( proxy.tmaster.server.config, @@ -865,12 +911,13 @@ class TestUpstreamProxySSL( setattr( proxy.tmaster.server.config, "check_%s" % attr, - HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns) + HostMatcher(handle, [".+:%s" % self.server.port] + old_matcher.patterns) ) setattr(self, "_ignore_%s_backup" % attr, backup) def _host_pattern_off(self, attr): + attr = "filter" if attr in ["allow", "ignore"] else attr backup = getattr(self, "_ignore_%s_backup" % attr) for proxy in reversed(self.chain): setattr( -- cgit v1.2.3 From 1b3f86e70938d43b5d1f702ef6ca18ecfa32c636 Mon Sep 17 00:00:00 2001 From: Pierre Gordon Date: Fri, 19 Apr 2019 13:10:39 -0500 Subject: Verify ignore_hosts & allow_hosts are mutually exclusive --- mitmproxy/options.py | 2 +- test/mitmproxy/proxy/test_config.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 9fc6966b..56146153 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -69,7 +69,7 @@ class Options(optmanager.OptManager): ) self.add_option( "allow_hosts", Sequence[str], [], - "Opposite of --ignore_hosts." + "Opposite of --ignore-hosts." ) self.add_option( "listen_host", str, "", diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 1da031c6..1319d1a9 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -17,3 +17,12 @@ class TestProxyConfig: opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")] with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): ProxyConfig(opts) + + def test_cannot_set_both_allow_and_filter_options(self): + opts = options.Options() + opts.ignore_hosts = ["foo"] + opts.allow_hosts = ["bar"] + with pytest.raises(exceptions.OptionsError, match="--ignore-hosts and --allow-hosts are " + "mutually exclusive; please choose " + "one."): + ProxyConfig(opts) -- cgit v1.2.3 From bcbf76a6281411b430639c58ca694bdc856fee72 Mon Sep 17 00:00:00 2001 From: Pierre Gordon Date: Fri, 19 Apr 2019 13:43:12 -0500 Subject: Only handle types "allow", "ignore" & "tcp" --- mitmproxy/proxy/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 55969d5e..75e372ae 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -25,10 +25,8 @@ class HostMatcher: host = "%s:%s" % address if self.handle in ["ignore", "tcp"]: return any(rex.search(host) for rex in self.regexes) - elif self.handle == "allow": + else: # self.handle == "allow" return any(not rex.search(host) for rex in self.regexes) - else: - return False def __bool__(self): return bool(self.patterns) -- cgit v1.2.3 From 1410bdd9ca4e969d9e7a11c7676a230f705e0564 Mon Sep 17 00:00:00 2001 From: ZHRhodes Date: Thu, 16 May 2019 23:21:47 -0700 Subject: Update howto-wireshark-tls.md Fixed a typo in describing the master keys --- docs/src/content/howto-wireshark-tls.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md index 588223ac..a55d177b 100644 --- a/docs/src/content/howto-wireshark-tls.md +++ b/docs/src/content/howto-wireshark-tls.md @@ -7,7 +7,7 @@ menu: # Wireshark and SSL/TLS Master Secrets -The SSL/SSL master keys can be logged by mitmproxy so that external programs can +The SSL/TLS master keys can be logged by mitmproxy so that external programs can decrypt SSL/TLS connections both from and to the proxy. Recent versions of Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. -- cgit v1.2.3 From 2fef56767b113953fb35b3b0729a064b92b8c305 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Mon, 27 May 2019 16:10:18 +0800 Subject: Use Brotli instead of brotlipy brotlipy is stuck at brotli 0.6 and upstream is inactive. Let's switch to the official binding which is up-to-date and has same interfaces. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4c17bdc6..7f83de63 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ setup( # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ "blinker>=1.4, <1.5", - "brotlipy>=0.7.0,<0.8", + "Brotli>=1.0,<1.1", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "click>=6.2, <7", "cryptography>=2.1.4,<2.5", -- cgit v1.2.3 From e63a099fcceeea90d6722243ae6bcd17ac5c3c31 Mon Sep 17 00:00:00 2001 From: Javier Tejero Date: Wed, 5 Jun 2019 14:23:35 +0200 Subject: Add refresh option to disable cookies --- mitmproxy/tools/cmdline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ad934ca2..458ef13a 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -85,6 +85,7 @@ def common_options(parser, opts): opts.make_parser(group, "server_replay", metavar="PATH", short="S") opts.make_parser(group, "server_replay_kill_extra") opts.make_parser(group, "server_replay_nopop") + opts.make_parser(group, "server_replay_refresh") # Replacements group = parser.add_argument_group("Replacements") -- cgit v1.2.3 From a4f8457dd3d47f09b57fd0251bd65c38fabdec1e Mon Sep 17 00:00:00 2001 From: Lucio Paiva Date: Wed, 26 Jun 2019 18:39:23 +0100 Subject: Minor fix re. macOS steps --- docs/src/content/howto-transparent.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 9be1e2f8..3915e4b7 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -229,7 +229,7 @@ for more. ### Work-around to redirect traffic originating from the machine itself -Follow the steps **1, 2** as above. In step **3** change the contents of the file **pf.conf** to +Follow steps **1, 2** as above, but in step **2** change the contents of the file **pf.conf** to {{< highlight none >}} #The ports to redirect to proxy @@ -257,7 +257,7 @@ rdr pass proto tcp from any to any port $redir_ports -> $tproxy pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user $redir_users {{< / highlight >}} -Follow steps **4-6** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **7** should look like: +Follow steps **3-5** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **6** should look like: {{< highlight bash >}} sudo -u nobody mitmproxy --mode transparent --showhost -- cgit v1.2.3 From 102e64bda562abf96f7efdb1078a8ad6131a4516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Border=C3=A9?= Date: Sat, 6 Jul 2019 14:28:30 +0200 Subject: websockets: replace masker with more optimized one --- mitmproxy/net/websockets/masker.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mitmproxy/net/websockets/masker.py b/mitmproxy/net/websockets/masker.py index 47b1a688..6134e09e 100644 --- a/mitmproxy/net/websockets/masker.py +++ b/mitmproxy/net/websockets/masker.py @@ -1,3 +1,6 @@ +import sys + + class Masker: """ Data sent from the server must be masked to prevent malicious clients @@ -12,12 +15,13 @@ class Masker: self.offset = 0 def mask(self, offset, data): - result = bytearray(data) - for i in range(len(data)): - result[i] ^= self.key[offset % 4] - offset += 1 - result = bytes(result) - return result + datalen = len(data) + offset_mod = offset % 4 + data = int.from_bytes(data, sys.byteorder) + num_keys = (datalen + offset_mod + 3) // 4 + mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen + + offset_mod], sys.byteorder) + return (data ^ mask).to_bytes(datalen, sys.byteorder) def __call__(self, data): ret = self.mask(self.offset, data) -- cgit v1.2.3 From 0e096137d590754fc926c162ab9798bbb1a661ef Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 17 Jul 2019 22:24:38 +0200 Subject: update pyinstaller --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 85cf40aa..3691eadf 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ commands = passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER deps = -rrequirements.txt - pyinstaller==3.4 + pyinstaller==3.5 twine==1.12.1 awscli commands = -- cgit v1.2.3 From b86cf6dea98b5d3d8c81898a2ecf18c647358edd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Jul 2019 02:52:07 +0200 Subject: fix py38 compat --- mitmproxy/platform/windows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index cb0a7096..19d9abd4 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -13,6 +13,7 @@ import typing import click import collections +import collections.abc import pydivert import pydivert.consts @@ -171,7 +172,7 @@ def MIB_TCPTABLE_OWNER_PID(size): TCP_TABLE_OWNER_PID_CONNECTIONS = 4 -class TcpConnectionTable(collections.Mapping): +class TcpConnectionTable(collections.abc.Mapping): DEFAULT_TABLE_SIZE = 4096 def __init__(self): -- cgit v1.2.3 From 2236b52816b53650e0f7204003a1292f724b682a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Jul 2019 01:54:39 +0200 Subject: fix sequence option declaration in yaml files --- mitmproxy/optmanager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 06e696c0..6e187b0d 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -320,7 +320,9 @@ class OptManager: update = {} for optname, optval in self.deferred.items(): if optname in self._options: - update[optname] = self.parse_setval(self._options[optname], optval) + if isinstance(optval, str): + optval = self.parse_setval(self._options[optname], optval) + update[optname] = optval self.update(**update) for k in update.keys(): del self.deferred[k] -- cgit v1.2.3 From a0e19336a2c99e062599719a5f8e5bcf8ddf7e7b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Jul 2019 03:14:46 +0200 Subject: remove unused dependency --- mitmproxy/tools/cmdline.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index a7820f76..21369a1f 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,8 +1,5 @@ import argparse -from mitmproxy.addons import core - - def common_options(parser, opts): parser.add_argument( '--version', -- cgit v1.2.3 From 9f69540d572791839afca071acfcbb987a298c6c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Jul 2019 17:54:33 +0200 Subject: fix addon deconstruction --- mitmproxy/addonmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 4214d6ea..8a565a73 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -184,7 +184,7 @@ class AddonManager: raise exceptions.AddonManagerError("No such addon: %s" % n) self.chain = [i for i in self.chain if i is not a] del self.lookup[_get_name(a)] - self.invoke_addon(a, "done") + self.invoke_addon(addon, "done") def __len__(self): return len(self.chain) -- cgit v1.2.3 From 9a159b439c24b547bbb6e62d60e0e9152a141be6 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Sun, 21 Jul 2019 10:33:59 -0400 Subject: Added Color palettes for lowlight, lowdark, solarized light, and solarized dark --- mitmproxy/tools/console/common.py | 3 ++ mitmproxy/tools/console/palettes.py | 70 +++++++++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 49f5c247..58a83c0e 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -309,6 +309,9 @@ def raw_format_flow(f): methods = { 'GET': 'method_get', 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' } uc = methods.get(f["req_method"], "method_other") if f['extended']: diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 405f1a6c..db73f0bd 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -23,7 +23,7 @@ class Palette: # List and Connections 'method', - 'method_get', 'method_post', 'method_other', 'method_http2_push', + 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', 'scheme_http', 'scheme_https', 'scheme_other', 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', @@ -122,8 +122,11 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), - method_get = ('dark cyan', 'default'), - method_post = ('dark red', 'default'), + method_get = ('light green', 'default'), + method_post = ('brown', 'default'), + method_delete = ('light red', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('dark red', 'default'), method_other = ('dark magenta', 'default'), method_http2_push = ('dark gray', 'default'), @@ -131,7 +134,7 @@ class LowDark(Palette): scheme_https = ('dark green', 'default'), scheme_other = ('dark magenta', 'default'), - url_punctuation = ('dark gray', 'default'), + url_punctuation = ('light gray', 'default'), url_domain = ('white', 'default'), url_filename = ('dark cyan', 'default'), url_extension = ('light gray', 'default'), @@ -219,6 +222,33 @@ class LowLight(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark green', 'default'), + method_post = ('brown', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('light red', 'default'), + method_delete = ('dark red', 'default'), + method_other = ('light magenta', 'default'), + method_http2_push = ('light gray','default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = ('light blue', 'default'), + url_query_value = ('dark blue', 'default'), + + content_none = ('black', 'default'), + content_text = ('dark gray', 'default'), + content_script = ('light green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('light red', 'default'), + content_other = ('light magenta', 'default'), + focus = ('black', 'default'), code_200 = ('dark green', 'default'), @@ -250,6 +280,7 @@ class LowLight(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('light gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Light(LowLight): @@ -308,7 +339,27 @@ class SolarizedLight(LowLight): option_active_selected = (sol_orange, sol_base2), # List and Connections - method = (sol_cyan, 'default'), + + method = ('dark cyan', 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_delete = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = ('light gray','default'), + + scheme_http = (sol_cyan, 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = (sol_blue, 'default'), + url_query_value = ('dark blue', 'default'), + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), @@ -363,10 +414,17 @@ class SolarizedDark(LowDark): option_active_selected = (sol_orange, sol_base00), # List and Connections - method = (sol_cyan, 'default'), method_http2_push = (sol_base01, 'default'), focus = (sol_base1, 'default'), + method = (sol_cyan, 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_delete = (sol_red, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + url_punctuation = ('h242', 'default'), url_domain = ('h252', 'default'), url_filename = ('h132', 'default'), -- cgit v1.2.3 From 998d150c1e91842a8cd7b1fc01cc6346aab2bfaa Mon Sep 17 00:00:00 2001 From: BkPHcgQL3V Date: Wed, 19 Dec 2018 17:52:32 +0000 Subject: Tabular list flow --- mitmproxy/tools/console/common.py | 360 ++++++++++++++++++++++++++++-------- mitmproxy/tools/console/flowlist.py | 2 +- mitmproxy/tools/console/flowview.py | 2 +- mitmproxy/tools/console/palettes.py | 67 ++++++- 4 files changed, 356 insertions(+), 75 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 5d7ee09d..49f5c247 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,5 +1,8 @@ import platform import typing +import datetime +import time +import math from functools import lru_cache import urwid @@ -97,12 +100,171 @@ if urwid.util.detected_encoding and not IS_WSL: SYMBOL_MARK = u"\u25cf" SYMBOL_UP = u"\u21E7" SYMBOL_DOWN = u"\u21E9" + SYMBOL_ELLIPSIS = u"\u2026" else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" SYMBOL_MARK = "[m]" SYMBOL_UP = "^" SYMBOL_DOWN = " " + SYMBOL_ELLIPSIS = "~" + + +def fixlen(s, maxlen): + if len(s) <= maxlen: + return s.ljust(maxlen) + else: + return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS + + +def fixlen_r(s, maxlen): + if len(s) <= maxlen: + return s.rjust(maxlen) + else: + return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):] + + +class TruncatedText(urwid.Widget): + def __init__(self, text, attr, align='left'): + self.text = text + self.attr = attr + self.align = align + super(TruncatedText, self).__init__() + + def pack(self, size, focus=False): + return (len(self.text), 1) + + def rows(self, size, focus=False): + return 1 + + def render(self, size, focus=False): + text = self.text + attr = self.attr + if self.align == 'right': + text = text[::-1] + attr = attr[::-1] + + text_len = len(text) # TODO: unicode? + if size is not None and len(size) > 0: + width = size[0] + else: + width = text_len + + if width >= text_len: + remaining = width - text_len + if remaining > 0: + c_text = text + ' ' * remaining + c_attr = attr + [('text', remaining)] + else: + c_text = text + c_attr = attr + else: + visible_len = width - len(SYMBOL_ELLIPSIS) + visible_text = text[0:visible_len] + c_text = visible_text + SYMBOL_ELLIPSIS + c_attr = (urwid.util.rle_subseg(attr, 0, len(visible_text.encode())) + + [('focus', len(SYMBOL_ELLIPSIS.encode()))]) + + if self.align == 'right': + c_text = c_text[::-1] + c_attr = c_attr[::-1] + + return urwid.TextCanvas([c_text.encode()], [c_attr], maxcol=width) + + +def truncated_plain(text, attr, align='left'): + return TruncatedText(text, [(attr, len(text.encode()))], align) + + +# Work around https://github.com/urwid/urwid/pull/330 +def rle_append_beginning_modify(rle, a_r): + """ + Append (a, r) (unpacked from *a_r*) to BEGINNING of rle. + Merge with first run when possible + + MODIFIES rle parameter contents. Returns None. + """ + a, r = a_r + if not rle: + rle[:] = [(a, r)] + else: + al, run = rle[0] + if a == al: + rle[0] = (a, run + r) + else: + rle[0:0] = [(a, r)] + + +def colorize_host(s): + if len(s) == 0 or s[0] == '[' or s.split('.')[-1].isdigit(): + main_part = -1 + else: + main_part = 1 # TODO: second-level domains (https://publicsuffix.org/list/) + part = 0 + attr = [] + for i in reversed(range(len(s))): + c = s[i] + if c == '.': + part += 1 + if c in ".:[]": + a = 'url_punctuation' + elif part == main_part: + a = 'url_domain' + else: + a = 'text' + rle_append_beginning_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_req(s): + path = s.split('?', 2)[0] + i_query = len(path) + i_last_slash = path.rfind('/') + i_ext = path[i_last_slash + 1:].rfind('.') + i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s) + in_val = False + attr = [] + for i in range(len(s)): + c = s[i] + if ((i < i_query and c == '/') or + (i < i_query and i > i_last_slash and c == '.') or + (i == i_query)): + a = 'url_punctuation' + elif i > i_query: + if in_val: + if c == '&': + in_val = False + a = 'url_punctuation' + else: + a = 'url_query_value' + else: + if c == '=': + in_val = True + a = 'url_punctuation' + else: + a = 'url_query_key' + elif i > i_ext: + a = 'url_extension' + elif i > i_last_slash: + a = 'url_filename' + else: + a = 'text' + urwid.util.rle_append_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_url(url): + parts = url.split('/', 3) + if len(parts) < 4 or len(parts[1]) > 0 or parts[0][-1:] != ':': + return [('error', len(url))] # bad URL + schemes = { + 'http:': 'scheme_http', + 'https:': 'scheme_https', + } + return [ + (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), + ('url_punctuation', 3), # :// + ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) @lru_cache(maxsize=800) @@ -110,50 +272,71 @@ def raw_format_flow(f): f = dict(f) pile = [] req = [] - if f["extended"]: - req.append( - fcol( - human.format_timestamp(f["req_timestamp"]), - "highlight" - ) - ) - else: - req.append(fcol(">>" if f["focus"] else " ", "focus")) - if f["marked"]: - req.append(fcol(SYMBOL_MARK, "mark")) + cursor = [' ', 'focus'] + if f.get('resp_is_replay', False): + cursor[0] = SYMBOL_REPLAY + cursor[1] = 'replay' + if f['marked']: + if cursor[0] == ' ': + cursor[0] = SYMBOL_MARK + cursor[1] = 'mark' + if f['focus']: + cursor[0] = '>' - if f["req_is_replay"]: - req.append(fcol(SYMBOL_REPLAY, "replay")) + req.append(fcol(*cursor)) - req.append(fcol(f["req_method"], "method")) + if f["two_line"]: + req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left')) + pile.append(urwid.Columns(req, dividechars=1)) - preamble = sum(i[1] for i in req) + len(req) - 1 + req = [] + req.append(fcol(' ', 'text')) if f["intercepted"] and not f["acked"]: uc = "intercept" - elif "resp_code" in f or "err_msg" in f: - uc = "text" + elif "resp_code" in f or f["err_msg"] is not None: + uc = "highlight" else: uc = "title" - url = f["req_url"] - - if f["max_url_len"] and len(url) > f["max_url_len"]: - url = url[:f["max_url_len"]] + "…" + if f["extended"]: + s = human.format_timestamp(f["req_timestamp"]) + else: + s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S") + req.append(fcol(s, uc)) + + methods = { + 'GET': 'method_get', + 'POST': 'method_post', + } + uc = methods.get(f["req_method"], "method_other") + if f['extended']: + req.append(fcol(f["req_method"], uc)) + if f["req_promise"]: + req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + else: + if f["req_promise"]: + uc = 'method_http2_push' + req.append(("fixed", 4, truncated_plain(f["req_method"], uc))) - if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): - url += " " + f["req_http_version"] - req.append( - urwid.Text([(uc, url)]) - ) + if f["two_line"]: + req.append(fcol(f["req_http_version"], 'text')) + else: + schemes = { + 'http': 'scheme_http', + 'https': 'scheme_https', + } + req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other"))) - pile.append(urwid.Columns(req, dividechars=1)) + req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right'))) + req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left'))) - resp = [] - resp.append( - ("fixed", preamble, urwid.Text("")) - ) + ret = (' ' * len(SYMBOL_RETURN), 'text') + status = ('', 'text') + content = ('', 'text') + size = ('', 'text') + duration = ('', 'text') if "resp_code" in f: codes = { @@ -163,79 +346,112 @@ def raw_format_flow(f): 5: "code_500", } ccol = codes.get(f["resp_code"] // 100, "code_other") - resp.append(fcol(SYMBOL_RETURN, ccol)) - if f["resp_is_replay"]: - resp.append(fcol(SYMBOL_REPLAY, "replay")) - resp.append(fcol(f["resp_code"], ccol)) - if f["extended"]: - resp.append(fcol(f["resp_reason"], ccol)) - if f["intercepted"] and f["resp_code"] and not f["acked"]: - rc = "intercept" + ret = (SYMBOL_RETURN, ccol) + status = (str(f["resp_code"]), ccol) + + if f["resp_len"] < 0: + if f["intercepted"] and f["resp_code"] and not f["acked"]: + rc = "intercept" + else: + rc = "content_none" + + if f["resp_len"] == -1: + contentdesc = "[content missing]" + else: + contentdesc = "[no content]" + content = (contentdesc, rc) else: - rc = "text" - - if f["resp_ctype"]: - resp.append(fcol(f["resp_ctype"], rc)) - resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["roundtrip"], rc)) + if f["resp_ctype"]: + ctype = f["resp_ctype"].split(";")[0] + if ctype.endswith('/javascript'): + rc = 'content_script' + elif ctype.startswith('text/'): + rc = 'content_text' + elif (ctype.startswith('image/') or + ctype.startswith('video/') or + ctype.startswith('font/') or + "/x-font-" in ctype): + rc = 'content_media' + elif ctype.endswith('/json') or ctype.endswith('/xml'): + rc = 'content_data' + elif ctype.startswith('application/'): + rc = 'content_raw' + else: + rc = 'content_other' + content = (ctype, rc) + + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99)) + + size_str = human.pretty_size(f["resp_len"]) + if not f['extended']: + # shorten to 5 chars max + if len(size_str) > 5: + size_str = size_str[0:4].rstrip('.') + size_str[-1:] + size = (size_str, rc) + + if f['duration'] is not None: + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99)) + duration = (human.pretty_duration(f['duration']), rc) elif f["err_msg"]: - resp.append(fcol(SYMBOL_RETURN, "error")) - resp.append( - urwid.Text([ - ( - "error", - f["err_msg"] - ) - ]) - ) - pile.append(urwid.Columns(resp, dividechars=1)) + status = ('Err', 'error') + content = f["err_msg"], 'error' + + if f["two_line"]: + req.append(fcol(*ret)) + req.append(fcol(fixlen(status[0], 3), status[1])) + req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right'))) + if f['extended']: + req.append(fcol(*size)) + else: + req.append(fcol(fixlen_r(size[0], 5), size[1])) + req.append(fcol(fixlen_r(duration[0], 5), duration[1])) + + pile.append(urwid.Columns(req, dividechars=1, min_width=15)) + return urwid.Pile(pile) -def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): +def format_flow(f, focus, extended=False, hostheader=False, cols=False): acked = False if f.reply and f.reply.state == "committed": acked = True - pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in f.metadata else '' d = dict( focus=focus, extended=extended, - max_url_len=max_url_len, + two_line=extended or cols < 100, intercepted=f.intercepted, acked=acked, req_timestamp=f.request.timestamp_start, req_is_replay=f.request.is_replay, - req_method=f.request.method + pushed, + req_method=f.request.method, + req_promise='h2-pushed-stream' in f.metadata, req_url=f.request.pretty_url if hostheader else f.request.url, + req_scheme=f.request.scheme, + req_host=f.request.pretty_host if hostheader else f.request.host, + req_path=f.request.path, req_http_version=f.request.http_version, err_msg=f.error.msg if f.error else None, marked=f.marked, ) if f.response: if f.response.raw_content: - contentdesc = human.pretty_size(len(f.response.raw_content)) + content_len = len(f.response.raw_content) elif f.response.raw_content is None: - contentdesc = "[content missing]" + content_len = -1 else: - contentdesc = "[no content]" - duration = 0 + content_len = -2 + duration = None if f.response.timestamp_end and f.request.timestamp_start: duration = f.response.timestamp_end - f.request.timestamp_start - roundtrip = human.pretty_duration(duration) d.update(dict( resp_code=f.response.status_code, resp_reason=f.response.reason, resp_is_replay=f.response.is_replay, - resp_clen=contentdesc, - roundtrip=roundtrip, + resp_len=content_len, + resp_ctype=f.response.headers.get("content-type"), + duration=duration, )) - t = f.response.headers.get("content-type") - if t: - d["resp_ctype"] = t.split(";")[0] - else: - d["resp_ctype"] = "" - return raw_format_flow(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index e947a582..63e67327 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -18,7 +18,7 @@ class FlowItem(urwid.WidgetWrap): self.flow, self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, ) def selectable(self): diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index b4e3876f..5466319a 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -38,7 +38,7 @@ class FlowViewHeader(urwid.WidgetWrap): False, extended=True, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 7930c4a3..405f1a6c 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -22,7 +22,12 @@ class Palette: 'option_selected_key', # List and Connections - 'method', 'focus', + 'method', + 'method_get', 'method_post', 'method_other', 'method_http2_push', + 'scheme_http', 'scheme_https', 'scheme_other', + 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', + 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', + 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', @@ -36,6 +41,7 @@ class Palette: # Commander 'commander_command', 'commander_invalid', 'commander_hint' ] + _fields.extend(['gradient_%02d' % i for i in range(100)]) high: typing.Mapping[str, typing.Sequence[str]] = None def palette(self, transparent): @@ -68,6 +74,27 @@ class Palette: return l +def gen_gradient(palette, cols): + for i in range(100): + palette['gradient_%02d' % i] = (cols[i * len(cols) // 100], 'default') + + +def gen_rgb_gradient(palette, cols): + parts = len(cols) - 1 + for i in range(100): + p = i / 100 + idx = int(p * parts) + t0 = cols[idx] + t1 = cols[idx + 1] + pp = p * parts % 1 + t = ( + round(t0[0] + (t1[0] - t0[0]) * pp), + round(t0[1] + (t1[1] - t0[1]) * pp), + round(t0[2] + (t1[2] - t0[2]) * pp), + ) + palette['gradient_%02d' % i] = ("#%x%x%x" % t, 'default') + + class LowDark(Palette): """ @@ -95,6 +122,30 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark cyan', 'default'), + method_post = ('dark red', 'default'), + method_other = ('dark magenta', 'default'), + method_http2_push = ('dark gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('dark green', 'default'), + scheme_other = ('dark magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('white', 'default'), + url_filename = ('dark cyan', 'default'), + url_extension = ('light gray', 'default'), + url_query_key = ('white', 'default'), + url_query_value = ('light gray', 'default'), + + content_none = ('dark gray', 'default'), + content_text = ('light gray', 'default'), + content_script = ('dark green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('dark red', 'default'), + content_other = ('dark magenta', 'default'), + focus = ('yellow', 'default'), code_200 = ('dark green', 'default'), @@ -127,6 +178,7 @@ class LowDark(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('dark gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Dark(LowDark): @@ -312,8 +364,20 @@ class SolarizedDark(LowDark): # List and Connections method = (sol_cyan, 'default'), + method_http2_push = (sol_base01, 'default'), focus = (sol_base1, 'default'), + url_punctuation = ('h242', 'default'), + url_domain = ('h252', 'default'), + url_filename = ('h132', 'default'), + url_extension = ('h96', 'default'), + url_query_key = ('h37', 'default'), + url_query_value = ('h30', 'default'), + + content_none = (sol_base01, 'default'), + content_text = (sol_base1, 'default'), + content_media = (sol_blue, 'default'), + code_200 = (sol_green, 'default'), code_300 = (sol_blue, 'default'), code_400 = (sol_orange, 'default',), @@ -342,6 +406,7 @@ class SolarizedDark(LowDark): commander_invalid = (sol_orange, 'default'), commander_hint = (sol_base00, 'default'), ) + gen_rgb_gradient(high, [(15, 0, 0), (15, 15, 0), (0, 15, 0), (0, 15, 15), (0, 0, 15)]) DEFAULT = "dark" -- cgit v1.2.3 From 826e42f793bc8def4d03e8cc435bf69fd2260dad Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Sun, 21 Jul 2019 10:33:59 -0400 Subject: Added Color palettes for lowlight, lowdark, solarized light, and solarized dark --- mitmproxy/tools/console/common.py | 3 ++ mitmproxy/tools/console/palettes.py | 70 +++++++++++++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 49f5c247..58a83c0e 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -309,6 +309,9 @@ def raw_format_flow(f): methods = { 'GET': 'method_get', 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' } uc = methods.get(f["req_method"], "method_other") if f['extended']: diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 405f1a6c..db73f0bd 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -23,7 +23,7 @@ class Palette: # List and Connections 'method', - 'method_get', 'method_post', 'method_other', 'method_http2_push', + 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', 'scheme_http', 'scheme_https', 'scheme_other', 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', @@ -122,8 +122,11 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), - method_get = ('dark cyan', 'default'), - method_post = ('dark red', 'default'), + method_get = ('light green', 'default'), + method_post = ('brown', 'default'), + method_delete = ('light red', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('dark red', 'default'), method_other = ('dark magenta', 'default'), method_http2_push = ('dark gray', 'default'), @@ -131,7 +134,7 @@ class LowDark(Palette): scheme_https = ('dark green', 'default'), scheme_other = ('dark magenta', 'default'), - url_punctuation = ('dark gray', 'default'), + url_punctuation = ('light gray', 'default'), url_domain = ('white', 'default'), url_filename = ('dark cyan', 'default'), url_extension = ('light gray', 'default'), @@ -219,6 +222,33 @@ class LowLight(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark green', 'default'), + method_post = ('brown', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('light red', 'default'), + method_delete = ('dark red', 'default'), + method_other = ('light magenta', 'default'), + method_http2_push = ('light gray','default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = ('light blue', 'default'), + url_query_value = ('dark blue', 'default'), + + content_none = ('black', 'default'), + content_text = ('dark gray', 'default'), + content_script = ('light green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('light red', 'default'), + content_other = ('light magenta', 'default'), + focus = ('black', 'default'), code_200 = ('dark green', 'default'), @@ -250,6 +280,7 @@ class LowLight(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('light gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Light(LowLight): @@ -308,7 +339,27 @@ class SolarizedLight(LowLight): option_active_selected = (sol_orange, sol_base2), # List and Connections - method = (sol_cyan, 'default'), + + method = ('dark cyan', 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_delete = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = ('light gray','default'), + + scheme_http = (sol_cyan, 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = (sol_blue, 'default'), + url_query_value = ('dark blue', 'default'), + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), @@ -363,10 +414,17 @@ class SolarizedDark(LowDark): option_active_selected = (sol_orange, sol_base00), # List and Connections - method = (sol_cyan, 'default'), method_http2_push = (sol_base01, 'default'), focus = (sol_base1, 'default'), + method = (sol_cyan, 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_delete = (sol_red, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + url_punctuation = ('h242', 'default'), url_domain = ('h252', 'default'), url_filename = ('h132', 'default'), -- cgit v1.2.3 From 2cc2fafd2e76b1f66cf2fa7ad93a5c4636eee6a8 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Sun, 21 Jul 2019 12:59:23 -0400 Subject: Added console_flowlist_layout option - default - list - table --- mitmproxy/tools/console/common.py | 105 +++++++++++++++++++++++++++++-- mitmproxy/tools/console/consoleaddons.py | 14 +++++ mitmproxy/tools/console/flowlist.py | 10 +++ mitmproxy/tools/console/flowview.py | 1 + mitmproxy/tools/console/palettes.py | 8 --- 5 files changed, 125 insertions(+), 13 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 58a83c0e..ee2e2a76 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -266,9 +266,95 @@ def colorize_url(url): ('url_punctuation', 3), # :// ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) +@lru_cache(maxsize=800) +def raw_format_list(f): + f = dict(f) + pile = [] + req = [] + if f["extended"]: + req.append( + fcol( + human.format_timestamp(f["req_timestamp"]), + "highlight" + ) + ) + else: + req.append(fcol(">>" if f["focus"] else " ", "focus")) + + if f["marked"]: + req.append(fcol(SYMBOL_MARK, "mark")) + + if f["req_is_replay"]: + req.append(fcol(SYMBOL_REPLAY, "replay")) + + req.append(fcol(f["req_method"], "method")) + + preamble = sum(i[1] for i in req) + len(req) - 1 + + if f["intercepted"] and not f["acked"]: + uc = "intercept" + elif "resp_code" in f or "err_msg" in f: + uc = "text" + else: + uc = "title" + + url = f["req_url"] + + if f["cols"] and len(url) > f["cols"]: + url = url[:f["cols"]] + "…" + + if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): + url += " " + f["req_http_version"] + req.append( + urwid.Text([(uc, url)]) + ) + + pile.append(urwid.Columns(req, dividechars=1)) + + resp = [] + resp.append( + ("fixed", preamble, urwid.Text("")) + ) + + if "resp_code" in f: + codes = { + 2: "code_200", + 3: "code_300", + 4: "code_400", + 5: "code_500", + } + ccol = codes.get(f["resp_code"] // 100, "code_other") + resp.append(fcol(SYMBOL_RETURN, ccol)) + if f["resp_is_replay"]: + resp.append(fcol(SYMBOL_REPLAY, "replay")) + resp.append(fcol(f["resp_code"], ccol)) + if f["extended"]: + resp.append(fcol(f["resp_reason"], ccol)) + if f["intercepted"] and f["resp_code"] and not f["acked"]: + rc = "intercept" + else: + rc = "text" + + if f["resp_ctype"]: + resp.append(fcol(f["resp_ctype"], rc)) + resp.append(fcol(f["resp_clen"], rc)) + resp.append(fcol(f["duration"], rc)) + + elif f["err_msg"]: + resp.append(fcol(SYMBOL_RETURN, "error")) + resp.append( + urwid.Text([ + ( + "error", + f["err_msg"] + ) + ]) + ) + pile.append(urwid.Columns(resp, dividechars=1)) + return urwid.Pile(pile) @lru_cache(maxsize=800) -def raw_format_flow(f): +def raw_format_table(f): f = dict(f) pile = [] req = [] @@ -415,14 +501,15 @@ def raw_format_flow(f): return urwid.Pile(pile) -def format_flow(f, focus, extended=False, hostheader=False, cols=False): +def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout='default'): acked = False if f.reply and f.reply.state == "committed": acked = True d = dict( focus=focus, extended=extended, - two_line=extended or cols < 100, + two_line=extended or cols < 80, + cols=cols, intercepted=f.intercepted, acked=acked, req_timestamp=f.request.timestamp_start, @@ -440,10 +527,14 @@ def format_flow(f, focus, extended=False, hostheader=False, cols=False): if f.response: if f.response.raw_content: content_len = len(f.response.raw_content) + contentdesc = human.pretty_size(len(f.response.raw_content)) elif f.response.raw_content is None: content_len = -1 + contentdesc = "[content missing]" else: content_len = -2 + contentdesc = "[no content]" + duration = None if f.response.timestamp_end and f.request.timestamp_start: duration = f.response.timestamp_end - f.request.timestamp_start @@ -454,7 +545,11 @@ def format_flow(f, focus, extended=False, hostheader=False, cols=False): resp_is_replay=f.response.is_replay, resp_len=content_len, resp_ctype=f.response.headers.get("content-type"), + resp_clen=contentdesc, duration=duration, - )) + )) - return raw_format_flow(tuple(sorted(d.items()))) + if ( (layout == 'default' and cols < 80) or layout == "list"): + return raw_format_list(tuple(sorted(d.items()))) + else: + return raw_format_table(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..0f7383f6 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -37,6 +37,12 @@ console_layouts = [ "horizontal", ] +console_flowlist_layout = [ + "default", + "table", + "list" +] + class UnsupportedLog: """ @@ -114,6 +120,14 @@ class ConsoleAddon: "Console mouse interaction." ) + loader.add_option( + "console_flowlist_layout", + str, "default", + "Set the flowlist layout", + choices=sorted(console_flowlist_layout) + + ) + @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: """ diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 63e67327..64c5e10c 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -19,6 +19,7 @@ class FlowItem(urwid.WidgetWrap): self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, cols=cols, + layout=self.master.options.console_flowlist_layout ) def selectable(self): @@ -84,6 +85,11 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): ) -> None: self.master: "mitmproxy.tools.console.master.ConsoleMaster" = master super().__init__(FlowListWalker(master)) + self.master.options.subscribe( + self.set_flowlist_layout, + ["console_flowlist_layout"] + ) + def keypress(self, size, key): if key == "m_start": @@ -96,3 +102,7 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): def view_changed(self): self.body.view_changed() + + def set_flowlist_layout(self, opts, updated): + self.master.ui.clear() + diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 5466319a..807c9714 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -39,6 +39,7 @@ class FlowViewHeader(urwid.WidgetWrap): extended=True, hostheader=self.master.options.showhost, cols=cols, + layout=self.master.options.console_flowlist_layout ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index a96689bf..2a7f1254 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -425,14 +425,6 @@ class SolarizedDark(LowDark): method_other = (sol_magenta, 'default'), method_http2_push = (sol_base01, 'default'), - method = (sol_cyan, 'default'), - method_get = (sol_green, 'default'), - method_post = (sol_orange, 'default'), - method_delete = (sol_red, 'default'), - method_head = (sol_cyan, 'default'), - method_put = (sol_red, 'default'), - method_other = (sol_magenta, 'default'), - url_punctuation = ('h242', 'default'), url_domain = ('h252', 'default'), url_filename = ('h132', 'default'), -- cgit v1.2.3 From df06c4da3b43f425f08afdfd3b12209f9fb84f01 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Sun, 21 Jul 2019 14:11:37 -0400 Subject: Fixed linting issues --- mitmproxy/addons/block.py | 2 +- mitmproxy/tools/cmdline.py | 1 + mitmproxy/tools/console/commandexecutor.py | 2 +- mitmproxy/tools/console/common.py | 6 ++++-- mitmproxy/tools/console/consoleaddons.py | 11 +++++------ mitmproxy/tools/console/flowlist.py | 2 -- mitmproxy/tools/console/palettes.py | 10 +++++----- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py index 91f9f709..4ccde0e1 100644 --- a/mitmproxy/addons/block.py +++ b/mitmproxy/addons/block.py @@ -36,4 +36,4 @@ class Block: layer.reply.kill() if ctx.options.block_global and address.is_global: ctx.log.warn("Client connection from %s killed by block_global" % astr) - layer.reply.kill() \ No newline at end of file + layer.reply.kill() diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 21369a1f..eb4a984d 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,5 +1,6 @@ import argparse + def common_options(parser, opts): parser.add_argument( '--version', diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py index 3db03d3e..c738e349 100644 --- a/mitmproxy/tools/console/commandexecutor.py +++ b/mitmproxy/tools/console/commandexecutor.py @@ -34,4 +34,4 @@ class CommandExecutor: ret, ), valign="top" - ) \ No newline at end of file + ) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index ee2e2a76..f13c876b 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -266,6 +266,7 @@ def colorize_url(url): ('url_punctuation', 3), # :// ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) + @lru_cache(maxsize=800) def raw_format_list(f): f = dict(f) @@ -353,6 +354,7 @@ def raw_format_list(f): pile.append(urwid.Columns(resp, dividechars=1)) return urwid.Pile(pile) + @lru_cache(maxsize=800) def raw_format_table(f): f = dict(f) @@ -547,9 +549,9 @@ def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout=' resp_ctype=f.response.headers.get("content-type"), resp_clen=contentdesc, duration=duration, - )) + )) - if ( (layout == 'default' and cols < 80) or layout == "list"): + if ((layout == 'default' and cols < 80) or layout == "list"): return raw_format_list(tuple(sorted(d.items()))) else: return raw_format_table(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index 0f7383f6..13f3ff7d 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -120,12 +120,11 @@ class ConsoleAddon: "Console mouse interaction." ) - loader.add_option( - "console_flowlist_layout", - str, "default", - "Set the flowlist layout", - choices=sorted(console_flowlist_layout) - + loader.add_option( + "console_flowlist_layout", + str, "default", + "Set the flowlist layout", + choices=sorted(console_flowlist_layout) ) @command.command("console.layout.options") diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 64c5e10c..9650c0d3 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -90,7 +90,6 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): ["console_flowlist_layout"] ) - def keypress(self, size, key): if key == "m_start": self.master.commands.execute("view.focus.go 0") @@ -105,4 +104,3 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): def set_flowlist_layout(self, opts, updated): self.master.ui.clear() - diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 2a7f1254..4eee7692 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -225,10 +225,10 @@ class LowLight(Palette): method_get = ('dark green', 'default'), method_post = ('brown', 'default'), method_head = ('dark cyan', 'default'), - method_put = ('light red', 'default'), + method_put = ('light red', 'default'), method_delete = ('dark red', 'default'), method_other = ('light magenta', 'default'), - method_http2_push = ('light gray','default'), + method_http2_push = ('light gray', 'default'), scheme_http = ('dark cyan', 'default'), scheme_https = ('light green', 'default'), @@ -344,10 +344,10 @@ class SolarizedLight(LowLight): method_get = (sol_green, 'default'), method_post = (sol_orange, 'default'), method_head = (sol_cyan, 'default'), - method_put = (sol_red, 'default'), + method_put = (sol_red, 'default'), method_delete = (sol_red, 'default'), method_other = (sol_magenta, 'default'), - method_http2_push = ('light gray','default'), + method_http2_push = ('light gray', 'default'), scheme_http = (sol_cyan, 'default'), scheme_https = ('light green', 'default'), @@ -359,7 +359,7 @@ class SolarizedLight(LowLight): url_extension = ('dark gray', 'default'), url_query_key = (sol_blue, 'default'), url_query_value = ('dark blue', 'default'), - + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), -- cgit v1.2.3 From 8a2a7652e48cba7ac8c4adadcfb23cce306fce85 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Mon, 22 Jul 2019 23:14:56 -0400 Subject: Edit URL now opens the external editor --- mitmproxy/tools/console/consoleaddons.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..6ec419fc 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -428,7 +428,12 @@ class ConsoleAddon: message.content = c.rstrip(b"\n") elif part == "set-cookies": self.master.switch_view("edit_focus_setcookies") - elif part in ["url", "method", "status_code", "reason"]: + elif part == "url": + url = flow.request.url.encode() + edited_url = self.master.spawn_editor(url) + url = edited_url.rstrip(b"\n") + flow.request.url = url.decode() + elif part in ["method", "status_code", "reason"]: self.master.commands.execute( "console.command flow.set @focus %s " % part ) -- cgit v1.2.3 From 586f8d2044e18f66b2a8ce531f94b659abfb8a1e Mon Sep 17 00:00:00 2001 From: king6cong Date: Tue, 30 Jul 2019 20:26:51 +0800 Subject: fix pf.conf on macOS --- docs/src/content/howto-transparent.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 3915e4b7..5f7c189a 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -50,7 +50,7 @@ a newly created `/etc/sysctl.d/mitmproxy.conf` (see [here](https://superuser.com sysctl -w net.ipv4.conf.all.send_redirects=0 {{< / highlight >}} -If your test device is on the same physical network, your machine shouldn't inform the device that +If your test device is on the same physical network, your machine shouldn't inform the device that there's a shorter route available by skipping the proxy. If you want to persist this across reboots, see above. @@ -83,7 +83,7 @@ The `--mode transparent` option turns on transparent mode, and the `--showhost` ### 5. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). @@ -132,7 +132,7 @@ mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). @@ -213,7 +213,7 @@ mitmproxy to use the value of the Host header for URL display. ### 7. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). {{% note %}} @@ -246,15 +246,9 @@ tproxy_user = "nobody" #This cannot involve the user which runs the #transparent proxy as that would cause an infinite loop. # -#Here we redirect for all users which don't run transparent proxy. -redir_users = "{ !=" $tproxy_user "}" - -#If you only wish to redirect traffic for particular users -#you may also do: -#redir_users = "{= john, = jane}" rdr pass proto tcp from any to any port $redir_ports -> $tproxy -pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user $redir_users +pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user { != $tproxy_user } {{< / highlight >}} Follow steps **3-5** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **6** should look like: -- cgit v1.2.3 From ba848e2040a50bc63fd14d79939df53869d9dcdb Mon Sep 17 00:00:00 2001 From: Tom Saleeba Date: Thu, 1 Aug 2019 15:37:25 +0930 Subject: docs: add instructions for transparent proxy on Linux for traffic originating from the mitmproxy host itself --- docs/src/content/howto-transparent.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 3915e4b7..5b4b5dd8 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -86,6 +86,29 @@ The `--mode transparent` option turns on transparent mode, and the `--showhost` Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). +### Work-around to redirect traffic originating from the machine itself + +Follow steps **1, 2** as above, but *instead* of the commands in step **3**, run the following + +Create a user to run the mitmproxy +{{< highlight bash >}} +sudo useradd --create-home mitmproxyuser +sudo -u mitmproxyuser 'cd ~ && pip install --user mitmproxy' +{{< / highlight >}} + +Then, configure the iptables rules to redirect all traffic from our local machine to mitmproxy. **Note**, as soon as you run these, you won't be able to perform successful network calls *until* you start mitmproxy. If you run into issues, `iptables -t nat -F` is a heavy handed way to flush (clear) *all* the rules from the iptables `nat` table (which includes any other rules you had configured). +{{< highlight bash >}} +iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 +iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 +ip6tables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 +ip6tables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 +{{< / highlight >}} + +This will redirect the packets from all users other than `mitmproxyuser` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `mitmproxyuser`. Hence step **4** should look like: +{{< highlight bash >}} +sudo -u mitmproxyuser bash -c '$HOME/.local/bin/mitmproxy --mode transparent --showhost --set block_global=false' +{{< / highlight >}} + ## OpenBSD -- cgit v1.2.3 From f47608c85ec0dcba456a759d1b862c3db8bcaddb Mon Sep 17 00:00:00 2001 From: Tom Saleeba Date: Thu, 1 Aug 2019 22:04:58 +0930 Subject: docs: correct command to install mitmproxy --- docs/src/content/howto-transparent.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 5b4b5dd8..803f8f10 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -91,12 +91,14 @@ Set the test device up to use the host on which mitmproxy is running as the defa Follow steps **1, 2** as above, but *instead* of the commands in step **3**, run the following Create a user to run the mitmproxy + {{< highlight bash >}} sudo useradd --create-home mitmproxyuser -sudo -u mitmproxyuser 'cd ~ && pip install --user mitmproxy' +sudo -u mitmproxyuser bash -c 'cd ~ && pip install --user mitmproxy' {{< / highlight >}} Then, configure the iptables rules to redirect all traffic from our local machine to mitmproxy. **Note**, as soon as you run these, you won't be able to perform successful network calls *until* you start mitmproxy. If you run into issues, `iptables -t nat -F` is a heavy handed way to flush (clear) *all* the rules from the iptables `nat` table (which includes any other rules you had configured). + {{< highlight bash >}} iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 -- cgit v1.2.3 From 0d042c6d8eb89691b7f731ccfd3d4bbaa27e3d48 Mon Sep 17 00:00:00 2001 From: cs Date: Fri, 9 Aug 2019 14:19:10 +0800 Subject: fix duplicate error response --- mitmproxy/proxy/protocol/http.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 2ae656b3..4c20617b 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -263,7 +263,7 @@ class HttpLayer(base.Layer): else: msg = "Unexpected CONNECT request." self.send_error_response(400, msg) - raise exceptions.ProtocolException(msg) + return False validate_request_form(self.mode, request) self.channel.ask("requestheaders", f) @@ -289,9 +289,12 @@ class HttpLayer(base.Layer): f.request = None f.error = flow.Error(str(e)) self.channel.ask("error", f) - raise exceptions.ProtocolException( - "HTTP protocol error in client request: {}".format(e) - ) from e + self.log( + "request", + "warn", + ["HTTP protocol error in client request: {}".format(e)] + ) + return False self.log("request", "debug", [repr(request)]) @@ -448,8 +451,8 @@ class HttpLayer(base.Layer): return False # should never be reached except (exceptions.ProtocolException, exceptions.NetlibException) as e: - self.send_error_response(502, repr(e)) if not f.response: + self.send_error_response(502, repr(e)) f.error = flow.Error(str(e)) self.channel.ask("error", f) return False -- cgit v1.2.3 From 4ce5e1386c7d065f4c4f8b68aa57b0e18d6945ca Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Tue, 3 Sep 2019 17:19:50 +0300 Subject: Updated har_dump,py timings dictionary function the HAR file spec (http://www.softwareishard.com/blog/har-12-spec/#timings) states that timings that do not apply for a certain requests should be set to -1, this example may set -1000 as a timings value for certain requests. This ends up producing invalid HAR files in many cases. My proposed fix is to assign -1 into the dic and only multiply by 1000 for other values --- examples/complex/har_dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 33a2f79f..e0964601 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -87,7 +87,7 @@ def response(flow): } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) + timings = dict([(k, -1 if v is -1 else int(1000 * v)) for k, v in timings_raw.items()]) # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. -- cgit v1.2.3 From dd3589ce345bcabe51563e1e1ac83797353baf8c Mon Sep 17 00:00:00 2001 From: Tero Saaristo Date: Thu, 5 Sep 2019 17:52:04 +0300 Subject: encoding: add support for zstd (zstandard) Handles zstandard-compressed bodies labeled as zstd. --- mitmproxy/addons/core.py | 2 +- mitmproxy/net/http/encoding.py | 24 ++++++++++++++++++++++-- mitmproxy/net/http/message.py | 2 +- mitmproxy/net/http/request.py | 2 +- setup.py | 1 + test/mitmproxy/net/http/test_encoding.py | 1 + 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index a908dbb3..5c9bbcd0 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -289,7 +289,7 @@ class Core: """ The possible values for an encoding specification. """ - return ["gzip", "deflate", "br"] + return ["gzip", "deflate", "br", "zstd"] @command.command("options.load") def options_load(self, path: mitmproxy.types.Path) -> None: diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py index 8cb96e5c..16d399ca 100644 --- a/mitmproxy/net/http/encoding.py +++ b/mitmproxy/net/http/encoding.py @@ -9,6 +9,7 @@ from io import BytesIO import gzip import zlib import brotli +import zstandard as zstd from typing import Union, Optional, AnyStr # noqa @@ -52,7 +53,7 @@ def decode( decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except TypeError: @@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except TypeError: @@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes: return brotli.compress(content) +def decode_zstd(content: bytes) -> bytes: + if not content: + return b"" + zstd_ctx = zstd.ZstdDecompressor() + try: + return zstd_ctx.decompress(content) + except zstd.ZstdError: + # If the zstd stream is streamed without a size header, + # try decoding with a 10MiB output buffer + return zstd_ctx.decompress(content, max_output_size=10 * 2**20) + + +def encode_zstd(content: bytes) -> bytes: + zstd_ctx = zstd.ZstdCompressor() + return zstd_ctx.compress(content) + + def decode_deflate(content: bytes) -> bytes: """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -170,6 +188,7 @@ custom_decode = { "gzip": decode_gzip, "deflate": decode_deflate, "br": decode_brotli, + "zstd": decode_zstd, } custom_encode = { "none": identity, @@ -177,6 +196,7 @@ custom_encode = { "gzip": encode_gzip, "deflate": encode_deflate, "br": encode_brotli, + "zstd": encode_zstd, } __all__ = ["encode", "decode"] diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 86782e8a..6830c6cd 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -236,7 +236,7 @@ class Message(serializable.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd33..3516e949 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -421,7 +421,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate", "br"} + for e in {"gzip", "identity", "deflate", "br", "zstd"} if e in accept_encoding ) ) diff --git a/setup.py b/setup.py index 7f83de63..12439f4e 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ setup( "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", "wsproto>=0.13.0,<0.14.0", + "zstandard>=0.11.0,<0.13.0", ], extras_require={ ':sys_platform == "win32"': [ diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py index 8dac12cb..7f768f39 100644 --- a/test/mitmproxy/net/http/test_encoding.py +++ b/test/mitmproxy/net/http/test_encoding.py @@ -19,6 +19,7 @@ def test_identity(encoder): 'gzip', 'br', 'deflate', + 'zstd', ]) def test_encoders(encoder): """ -- cgit v1.2.3 From ebfff8db761bf101ae46ac9ca0983408897baad2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 5 Sep 2019 21:36:28 +0200 Subject: lint --- mitmproxy/tools/cmdline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 21369a1f..eb4a984d 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,5 +1,6 @@ import argparse + def common_options(parser, opts): parser.add_argument( '--version', -- cgit v1.2.3 From e77f375186403cee05dc66c069ed1155059b2d63 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 5 Sep 2019 22:08:18 +0200 Subject: lint --- .travis.yml | 1 - mitmproxy/contentviews/base.py | 2 +- mitmproxy/contentviews/image/image_parser.py | 2 +- setup.py | 2 +- test/mitmproxy/addons/test_view.py | 2 +- test/mitmproxy/script/test_concurrent.py | 26 +++++++++++++------------- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 20afc279..44d452c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,6 @@ matrix: - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - sudo dpkg -i hugo*.deb - pip install tox virtualenv setuptools - - pyenv global system 3.6 script: - tox after_success: diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 6072dfb7..9b34f3d4 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -37,7 +37,7 @@ class View: def format_pairs( items: typing.Iterable[typing.Tuple[TTextType, TTextType]] -)-> typing.Iterator[TViewLine]: +) -> typing.Iterator[TViewLine]: """ Helper function that accepts a list of (k,v) pairs into a list of diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index fcc50cb5..d5bb404f 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata: entries = block.body.body.entries for entry in entries: comment = entry.bytes - if comment is not b'': + if comment != b'': parts.append(('comment', str(comment))) return parts diff --git a/setup.py b/setup.py index 7f83de63..50d02a55 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.5,<3.7", + "flake8>=3.5,<=3.7.8", "Flask>=1.0,<1.1", "mypy>=0.590,<0.591", "parver>=0.1,<2.0", diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 976c14b7..f5088a68 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -471,7 +471,7 @@ def test_focus(): v = view.View() v.add([tft()]) f = view.Focus(v) - assert f.index is 0 + assert f.index == 0 assert f.flow is v[0] # Start empty diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 3ec58760..70d41511 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest): assert await tctx.master.await_log("decorator not supported") def test_concurrent_class(self, tdata): - with taddons.context() as tctx: - sc = tctx.script( - tdata.path( - "mitmproxy/data/addonscripts/concurrent_decorator_class.py" - ) + with taddons.context() as tctx: + sc = tctx.script( + tdata.path( + "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) - f1, f2 = tflow.tflow(), tflow.tflow() - tctx.cycle(sc, f1) - tctx.cycle(sc, f2) - start = time.time() - while time.time() - start < 5: - if f1.reply.state == f2.reply.state == "committed": - return - raise ValueError("Script never acked") + ) + f1, f2 = tflow.tflow(), tflow.tflow() + tctx.cycle(sc, f1) + tctx.cycle(sc, f2) + start = time.time() + while time.time() - start < 5: + if f1.reply.state == f2.reply.state == "committed": + return + raise ValueError("Script never acked") -- cgit v1.2.3 From e97a804e89454f5f5f546f3f99635ca8b99d75d3 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 5 Sep 2019 22:13:49 +0200 Subject: make dict comprehension more readable --- examples/complex/har_dump.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index e0964601..414b4f61 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -87,7 +87,10 @@ def response(flow): } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, -1 if v is -1 else int(1000 * v)) for k, v in timings_raw.items()]) + timings = { + k: int(1000 * v) if v != -1 else -1 + for k, v in timings_raw.items() + } # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. -- cgit v1.2.3 From 6ba39ede08f969f17371f4a2fd050155be9e483d Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 24 Sep 2019 14:59:26 +0300 Subject: Update concepts-certificates.md Update to direct people to using Safari instead of another browser on iOS. --- docs/src/content/concepts-certificates.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 88482047..4e2ae47a 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -24,6 +24,9 @@ something like this: Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. +Note: If you are using an iOS device, you should be using the Safari browser +so that it opens the proper prompts for installing the certificate. + ## Installing the mitmproxy CA certificate manually Sometimes using the quick install app is not an option - Java or the iOS -- cgit v1.2.3 From 94ca23b78208c5976a4c013a1acdd4c37e83ba76 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Tue, 24 Sep 2019 22:29:56 -0400 Subject: TLD and SLD are now highlighted using publicsuffix - Added time.time() as the default for the start time on fake requests --- mitmproxy/net/http/request.py | 2 ++ mitmproxy/tools/console/common.py | 40 ++++++++++++++++++++++----------------- setup.py | 1 + 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd33..6ca3973f 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,5 +1,6 @@ import re import urllib +import time from typing import Optional, AnyStr, Dict, Iterable, Tuple, Union from mitmproxy.coretypes import multidict @@ -101,6 +102,7 @@ class Request(message.Message): ) req.url = url + req.timestamp_start = time.time() # Headers can be list or dict, we differentiate here. if isinstance(headers, dict): diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index f13c876b..527756c1 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -4,6 +4,7 @@ import datetime import time import math from functools import lru_cache +from publicsuffix2 import get_sld, get_tld import urwid import urwid.util @@ -195,24 +196,29 @@ def rle_append_beginning_modify(rle, a_r): rle[0:0] = [(a, r)] -def colorize_host(s): - if len(s) == 0 or s[0] == '[' or s.split('.')[-1].isdigit(): - main_part = -1 - else: - main_part = 1 # TODO: second-level domains (https://publicsuffix.org/list/) - part = 0 +def colorize_host(host): + tld = get_tld(host) + sld = get_sld(host) + attr = [] - for i in reversed(range(len(s))): - c = s[i] - if c == '.': - part += 1 - if c in ".:[]": - a = 'url_punctuation' - elif part == main_part: - a = 'url_domain' + + tld_size = len(tld) + sld_size = len(sld) - tld_size + + for letter in reversed(range(len(host))): + character = host[letter] + if tld_size > 0: + style = 'url_domain' + tld_size -= 1 + elif tld_size == 0: + style = 'text' + tld_size -= 1 + elif sld_size > 0: + sld_size -= 1 + style = 'url_extension' else: - a = 'text' - rle_append_beginning_modify(attr, (a, len(c.encode()))) + style = 'text' + rle_append_beginning_modify(attr, (style, len(character.encode()))) return attr @@ -510,7 +516,7 @@ def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout=' d = dict( focus=focus, extended=extended, - two_line=extended or cols < 80, + two_line=extended or cols < 100, cols=cols, intercepted=f.intercepted, acked=acked, diff --git a/setup.py b/setup.py index 7f83de63..52becc00 100644 --- a/setup.py +++ b/setup.py @@ -81,6 +81,7 @@ setup( "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", "wsproto>=0.13.0,<0.14.0", + "publicsuffix2~=2.20" ], extras_require={ ':sys_platform == "win32"': [ -- cgit v1.2.3 From c02d515b2aada5aa63439c6f2d71ae86a7c4dc6b Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Tue, 24 Sep 2019 23:36:39 -0400 Subject: Fixed Duration on list view - Set min size to 100 for table view --- mitmproxy/tools/console/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 527756c1..43ab50cb 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -345,7 +345,8 @@ def raw_format_list(f): if f["resp_ctype"]: resp.append(fcol(f["resp_ctype"], rc)) resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["duration"], rc)) + pretty_duration = human.pretty_duration(f["duration"]) + resp.append(fcol(pretty_duration, rc)) elif f["err_msg"]: resp.append(fcol(SYMBOL_RETURN, "error")) @@ -557,7 +558,7 @@ def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout=' duration=duration, )) - if ((layout == 'default' and cols < 80) or layout == "list"): + if ((layout == 'default' and cols < 100) or layout == "list"): return raw_format_list(tuple(sorted(d.items()))) else: return raw_format_table(tuple(sorted(d.items()))) -- cgit v1.2.3 From 55719a6942e49ffe82d1fcce7d837a8e5068d904 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Tue, 24 Sep 2019 23:47:14 -0400 Subject: Updated Setup.py A comma broke the build --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b389fbdf..91a14fe8 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ setup( "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", "wsproto>=0.13.0,<0.14.0", - "publicsuffix2~=2.20" + "publicsuffix2~=2.20", "zstandard>=0.11.0,<0.13.0", ], extras_require={ -- cgit v1.2.3 From 86aacf4d5bf987f1b04b6d02c8cd63b32dda051b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferenc=20so=C3=B3s?= Date: Fri, 27 Sep 2019 10:00:50 +0200 Subject: ssl_insecure added to option_whitelist because it was missing --- mitmproxy/tools/web/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 6e6b6223..6bfce34e 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -432,7 +432,7 @@ class Settings(RequestHandler): def put(self): update = self.json option_whitelist = { - "intercept", "showhost", "upstream_cert", + "intercept", "showhost", "upstream_cert", "ssl_insecure", "rawtcp", "http2", "websocket", "anticache", "anticomp", "stickycookie", "stickyauth", "stream_large_bodies" } -- cgit v1.2.3 From a54954ee1ebb46cd4e163af407faf1e034e1a4e6 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 28 Sep 2019 12:01:39 +0200 Subject: fix linting --- examples/complex/har_dump.py | 2 +- setup.cfg | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 414b4f61..e3cea9fd 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -88,7 +88,7 @@ def response(flow): # HAR timings are integers in ms, so we re-encode the raw timings to that format. timings = { - k: int(1000 * v) if v != -1 else -1 + k: int(1000 * v) if v != -1 else -1 for k, v in timings_raw.items() } diff --git a/setup.cfg b/setup.cfg index 83144c22..c717bd1c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,7 +59,6 @@ exclude = mitmproxy/net/http/headers.py mitmproxy/net/http/message.py mitmproxy/net/http/multipart.py - mitmproxy/net/http/url.py mitmproxy/net/tcp.py mitmproxy/net/tls.py mitmproxy/options.py -- cgit v1.2.3 From b4ffc929a2d0216c3a0910e633f31212c12136cd Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 28 Sep 2019 12:29:07 +0200 Subject: bump dependencies --- .gitignore | 1 + .travis.yml | 6 +++--- setup.py | 14 +++++++------- tox.ini | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index deb93814..6b514131 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ MANIFEST *.egg-info/ .coverage* .idea +.vscode .cache/ .tox*/ build/ diff --git a/.travis.yml b/.travis.yml index 44d452c2..2e375078 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,9 +64,9 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.5 - pyenv global 3.6.5 - pyenv shell 3.6.5 + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.9 + pyenv global 3.6.9 + pyenv shell 3.6.9 fi - pip install tox virtualenv setuptools diff --git a/setup.py b/setup.py index 91a14fe8..f1bc2efb 100644 --- a/setup.py +++ b/setup.py @@ -63,24 +63,24 @@ setup( install_requires=[ "blinker>=1.4, <1.5", "Brotli>=1.0,<1.1", - "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! + "certifi>=2019.9.11", # no semver here - this should always be on the last release! "click>=6.2, <7", "cryptography>=2.1.4,<2.5", "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", - "ldap3>=2.5,<2.6", + "ldap3>=2.6.1,<2.7", "passlib>=1.6.5, <1.8", - "protobuf>=3.6.0, <3.7", + "protobuf>=3.6.0, <3.10", "pyasn1>=0.3.1,<0.5", - "pyOpenSSL>=17.5,<18.1", - "pyparsing>=2.1.3,<2.4", + "pyOpenSSL>=19.0.0,<20", + "pyparsing>=2.4.2,<2.5", "pyperclip>=1.6.0,<1.8", "ruamel.yaml>=0.15,<0.16", "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", - "wsproto>=0.13.0,<0.14.0", + "wsproto>=0.14.0,<0.15.0", "publicsuffix2~=2.20", "zstandard>=0.11.0,<0.13.0", ], @@ -91,7 +91,7 @@ setup( 'dev': [ "asynctest>=0.12.0", "flake8>=3.5,<=3.7.8", - "Flask>=1.0,<1.1", + "Flask>=1.0,<1.2", "mypy>=0.590,<0.591", "parver>=0.1,<2.0", "pytest-asyncio>=0.8", diff --git a/tox.ini b/tox.ini index 3691eadf..8efe04f5 100644 --- a/tox.ini +++ b/tox.ini @@ -45,7 +45,7 @@ passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYIN deps = -rrequirements.txt pyinstaller==3.5 - twine==1.12.1 + twine==2.0.0 awscli commands = mitmdump --version -- cgit v1.2.3 From 53cb5bf40f40cef2c5b4e05e7be42949146a3f58 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 28 Sep 2019 16:53:24 +0200 Subject: bump deps --- .travis.yml | 4 ++-- examples/__init__.py | 0 examples/complex/__init__.py | 0 setup.py | 6 +++--- test/full_coverage_plugin.py | 8 ++++---- test/mitmproxy/net/test_tls.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 examples/__init__.py create mode 100644 examples/complex/__init__.py diff --git a/.travis.yml b/.travis.yml index 2e375078..035efb79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,7 +51,7 @@ matrix: install: - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - sudo dpkg -i hugo*.deb - - pip install tox virtualenv setuptools + - pip install -U tox virtualenv setuptools script: - tox after_success: @@ -68,7 +68,7 @@ install: pyenv global 3.6.9 pyenv shell 3.6.9 fi - - pip install tox virtualenv setuptools + - pip install -U tox virtualenv setuptools script: # All these steps MUST succeed for the job to be successful! diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/complex/__init__.py b/examples/complex/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/setup.py b/setup.py index 180058a2..a40df8f7 100644 --- a/setup.py +++ b/setup.py @@ -76,8 +76,8 @@ setup( "pyOpenSSL>=19.0.0,<20", "pyparsing>=2.4.2,<2.5", "pyperclip>=1.6.0,<1.8", - "ruamel.yaml>=0.15,<0.16", - "sortedcontainers>=1.5.4,<2.1", + "ruamel.yaml>=0.16,<0.17", + "sortedcontainers>=2.1.0,<2.2", "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", "wsproto>=0.14.0,<0.15.0", @@ -101,7 +101,7 @@ setup( "pytest-xdist>=1.22,<2", "pytest>=4.0,<5", "requests>=2.9.1, <3", - "tox>=3.5,<3.6", + "tox>=3.5,<3.15", "rstcheck>=2.2, <4.0", ], 'examples': [ diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py index ec30a9f9..f99dc7ab 100644 --- a/test/full_coverage_plugin.py +++ b/test/full_coverage_plugin.py @@ -55,7 +55,7 @@ def pytest_runtestloop(session): yield return - cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov + cov = session.config.pluginmanager.getplugin("_cov").cov_controller.cov if os.name == 'nt': cov.exclude('pragma: windows no cover') @@ -68,7 +68,7 @@ def pytest_runtestloop(session): yield - coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov]) + coverage_values = dict([(name, 0) for name in session.config.option.full_cov]) prefix = os.getcwd() @@ -92,7 +92,7 @@ def pytest_runtestloop(session): coverage_passed = False -def pytest_terminal_summary(terminalreporter, exitstatus): +def pytest_terminal_summary(terminalreporter, exitstatus, config): global enable_coverage global coverage_values global coverage_passed @@ -119,7 +119,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus): terminalreporter.write(msg, **markup) else: msg = 'SUCCESS: Full test coverage reached in modules and files:\n' - msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov)) + msg += '{}\n\n'.format('\n'.join(config.option.full_cov)) terminalreporter.write(msg, green=True) msg = '\nExcluded files:\n' diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index 489bf89f..68e67dbe 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -87,13 +87,13 @@ def test_get_client_hello(): rfile = io.BufferedReader(io.BytesIO( FULL_CLIENT_HELLO_NO_EXTENSIONS[:30] )) - with pytest.raises(exceptions.TlsProtocolException, message="Unexpected EOF"): + with pytest.raises(exceptions.TlsProtocolException, match="Unexpected EOF"): tls.get_client_hello(rfile) rfile = io.BufferedReader(io.BytesIO( b"GET /" )) - with pytest.raises(exceptions.TlsProtocolException, message="Expected TLS record"): + with pytest.raises(exceptions.TlsProtocolException, match="Expected TLS record"): tls.get_client_hello(rfile) @@ -153,5 +153,5 @@ class TestClientHello: b"\x01\x00\x00\x03" + # handshake header b"foo" )) - with pytest.raises(exceptions.TlsProtocolException, message='Cannot parse Client Hello'): + with pytest.raises(exceptions.TlsProtocolException, match='Cannot parse Client Hello'): tls.ClientHello.from_file(rfile) -- cgit v1.2.3 From ace79afefc7b9d8055a043a3a80d4cb8cee8a59e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 28 Sep 2019 19:52:18 +0200 Subject: bump more deps --- setup.py | 21 ++++++++++----------- test/full_coverage_plugin.py | 4 ++-- test/individual_coverage.py | 12 ++++++------ tox.ini | 2 +- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index a40df8f7..83981efb 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ setup( "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", "wsproto>=0.14.0,<0.15.0", - "publicsuffix2~=2.20", + "publicsuffix2>=2.20190812,<3", "zstandard>=0.11.0,<0.13.0", ], extras_require={ @@ -90,22 +90,21 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.5,<=3.7.8", + "flake8>=3.7.8,<3.8", "Flask>=1.0,<1.2", "mypy>=0.590,<0.591", "parver>=0.1,<2.0", - "pytest-asyncio>=0.8", - "pytest-cov>=2.5.1,<3", - "pytest-faulthandler>=1.3.1,<2", - "pytest-timeout>=1.2.1,<2", - "pytest-xdist>=1.22,<2", - "pytest>=4.0,<5", - "requests>=2.9.1, <3", + "pytest-asyncio>=0.10.0,<0.11", + "pytest-cov>=2.7.1,<3", + "pytest-timeout>=1.3.3,<2", + "pytest-xdist>=1.29,<2", + "pytest>=5.1.3,<6", + "requests>=2.9.1,<3", "tox>=3.5,<3.15", - "rstcheck>=2.2, <4.0", + "rstcheck>=2.2,<4.0", ], 'examples': [ - "beautifulsoup4>=4.4.1, <4.7" + "beautifulsoup4>=4.4.1,<4.7" ] } ) diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py index f99dc7ab..ab1206ea 100644 --- a/test/full_coverage_plugin.py +++ b/test/full_coverage_plugin.py @@ -31,8 +31,8 @@ def pytest_configure(config): global no_full_cov enable_coverage = ( - len(config.getoption('file_or_dir')) == 0 and - len(config.getoption('full_cov')) > 0 and + config.getoption('file_or_dir') and len(config.getoption('file_or_dir')) == 0 and + config.getoption('full_cov') and len(config.getoption('full_cov')) > 0 and config.pluginmanager.getplugin("_cov") is not None and config.pluginmanager.getplugin("_cov").cov_controller is not None and config.pluginmanager.getplugin("_cov").cov_controller.cov is not None diff --git a/test/individual_coverage.py b/test/individual_coverage.py index 097b290f..54180eac 100755 --- a/test/individual_coverage.py +++ b/test/individual_coverage.py @@ -19,29 +19,29 @@ def run_tests(src, test, fail): e = pytest.main([ '-qq', '--disable-pytest-warnings', - '--no-faulthandler', '--cov', src.replace('.py', '').replace('/', '.'), '--cov-fail-under', '100', '--cov-report', 'term-missing:skip-covered', + '-o', 'faulthandler_timeout=0', test ]) if e == 0: if fail: - print("UNEXPECTED SUCCESS:", src, "Please remove this file from setup.cfg tool:individual_coverage/exclude.") + print("FAIL DUE TO UNEXPECTED SUCCESS:", src, "Please remove this file from setup.cfg tool:individual_coverage/exclude.") e = 42 else: - print("SUCCESS: ", src) + print("Success:", src) else: if fail: - print("IGNORING FAIL: ", src) + print("Ignoring allowed fail:", src) e = 0 else: cov = [l for l in stdout.getvalue().split("\n") if (src in l) or ("was never imported" in l)] if len(cov) == 1: - print("FAIL: ", cov[0]) + print("FAIL:", cov[0]) else: - print("FAIL: ", src, test, stdout.getvalue(), stdout.getvalue()) + print("FAIL:", src, test, stdout.getvalue(), stdout.getvalue()) print(stderr.getvalue()) print(stdout.getvalue()) diff --git a/tox.ini b/tox.ini index 8efe04f5..8c8fbaa2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, lint +envlist = py35, py36, py37, lint, individual_coverage, wheeltest, docs skipsdist = True toxworkdir={env:TOX_WORK_DIR:.tox} -- cgit v1.2.3 From 7ef91f46a32808bf6226a993e06d0a6522a663d5 Mon Sep 17 00:00:00 2001 From: Nirusu Date: Mon, 30 Sep 2019 11:41:49 +0200 Subject: Update howto-transparent-vms.md for newer versions (#3597) Update howto-transparent-vms.md for newer versions --- docs/src/content/howto-transparent-vms.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/src/content/howto-transparent-vms.md b/docs/src/content/howto-transparent-vms.md index 1446ede7..f251bc44 100644 --- a/docs/src/content/howto-transparent-vms.md +++ b/docs/src/content/howto-transparent-vms.md @@ -14,9 +14,13 @@ Internal Network* setup can be applied to other setups. ## 1. Configure Proxy VM -On the proxy machine, **eth0** is connected to the internet. **eth1** is -connected to the internal network that will be proxified and configured -to use a static ip (192.168.3.1). +First, we have to find out under which name Ubuntu has mapped our network interfaces. You can find this information with: + +{{< highlight bash >}} +ip link +{{< / highlight >}} + +Usually with Ubuntu and Virtualbox, **eth0** or **enp0s3** (Ubuntu 15.10 and newer) is connected to the internet and **eth1** or **enp0s8** (Ubuntu 15.10 and newer) is connected to the internal network that will be proxified and configured to use a static ip (192.168.3.1). If the names differ, use the ones you got from the *ip link* command. ### VirtualBox configuration @@ -65,6 +69,7 @@ Replace **/etc/dnsmasq.conf** with the following configuration: {{< highlight none >}} # Listen for DNS requests on the internal network interface=eth1 +bind-interfaces # Act as a DHCP server, assign IP addresses to clients dhcp-range=192.168.3.10,192.168.3.100,96h # Broadcast gateway and dns server information @@ -93,10 +98,11 @@ IP address via DHCP: ## 3. Redirect traffic to mitmproxy -To redirect traffic to mitmproxy, we need to add two iptables +To redirect traffic to mitmproxy, we need to enable IP forwarding and add two iptables rules: {{< highlight bash >}} +sudo sysctl -w net.ipv4.ip_forward=1 sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080 sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080 {{< / highlight >}} -- cgit v1.2.3 From 93f9e30728300cc5379fd90171818f642d8f24aa Mon Sep 17 00:00:00 2001 From: vin01 <30344579+vin01@users.noreply.github.com> Date: Mon, 30 Sep 2019 17:19:52 +0000 Subject: Add key_size option to define rsa key size (#3657) --- mitmproxy/certs.py | 12 ++++++------ mitmproxy/options.py | 7 +++++++ mitmproxy/proxy/config.py | 4 +++- mitmproxy/tools/cmdline.py | 1 + pathod/pathod.py | 4 +++- test/mitmproxy/test_certs.py | 20 ++++++++++---------- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 6f5f8c09..0ec63b50 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -36,9 +36,9 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= """ -def create_ca(organization, cn, exp): +def create_ca(organization, cn, exp, key_size): key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) + key.generate_key(OpenSSL.crypto.TYPE_RSA, key_size) cert = OpenSSL.crypto.X509() cert.set_serial_number(int(time.time() * 10000)) cert.set_version(2) @@ -182,10 +182,10 @@ class CertStore: return dh @classmethod - def from_store(cls, path, basename): + def from_store(cls, path, basename, key_size): ca_path = os.path.join(path, basename + "-ca.pem") if not os.path.exists(ca_path): - key, ca = cls.create_store(path, basename) + key, ca = cls.create_store(path, basename, key_size) else: with open(ca_path, "rb") as f: raw = f.read() @@ -215,14 +215,14 @@ class CertStore: os.umask(original_umask) @staticmethod - def create_store(path, basename, organization=None, cn=None, expiry=DEFAULT_EXP): + def create_store(path, basename, key_size, organization=None, cn=None, expiry=DEFAULT_EXP): if not os.path.exists(path): os.makedirs(path) organization = organization or basename cn = cn or basename - key, ca = create_ca(organization=organization, cn=cn, exp=expiry) + key, ca = create_ca(organization=organization, cn=cn, exp=expiry, key_size=key_size) # Dump the CA plus private key with CertStore.umask_secret(), open(os.path.join(path, basename + "-ca.pem"), "wb") as f: f.write( diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 56146153..69ffd033 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -7,6 +7,7 @@ from mitmproxy.net import tls CONF_DIR = "~/.mitmproxy" LISTEN_PORT = 8080 CONTENT_VIEW_LINES_CUTOFF = 512 +KEY_SIZE = 2048 class Options(optmanager.OptManager): @@ -173,5 +174,11 @@ class Options(optmanager.OptManager): speedup flows browsing. """ ) + self.add_option( + "key_size", int, KEY_SIZE, + """ + TLS key size for certificates and CA. + """ + ) self.update(**kwargs) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 75e372ae..881ed2fd 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -64,9 +64,11 @@ class ProxyConfig: "Certificate Authority parent directory does not exist: %s" % os.path.dirname(certstore_path) ) + key_size = options.key_size self.certstore = certs.CertStore.from_store( certstore_path, - CONF_BASENAME + CONF_BASENAME, + key_size ) for c in options.certs: diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index eb9147b3..e9ff973f 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -68,6 +68,7 @@ def common_options(parser, opts): group = parser.add_argument_group("SSL") opts.make_parser(group, "certs", metavar="SPEC") opts.make_parser(group, "ssl_insecure", short="k") + opts.make_parser(group, "key_size", metavar="KEY_SIZE") # Client replay group = parser.add_argument_group("Client Replay") diff --git a/pathod/pathod.py b/pathod/pathod.py index b330a293..3fe2f901 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -21,6 +21,7 @@ CONFDIR = "~/.mitmproxy" CERTSTORE_BASENAME = "mitmproxy" CA_CERT_NAME = "mitmproxy-ca.pem" DEFAULT_CRAFT_ANCHOR = "/p/" +KEY_SIZE = 2048 logger = logging.getLogger('pathod') @@ -54,7 +55,8 @@ class SSLOptions: self.alpn_select = alpn_select self.certstore = mcerts.CertStore.from_store( os.path.expanduser(confdir), - CERTSTORE_BASENAME + CERTSTORE_BASENAME, + KEY_SIZE ) for i in certs or []: self.certstore.add_cert_file(*i) diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index b8ad1d36..37604f54 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -35,20 +35,20 @@ from ..conftest import skip_windows class TestCertStore: def test_create_explicit(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca.get_cert(b"foo", []) - ca2 = certs.CertStore.from_store(str(tmpdir), "test") + ca2 = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca2.get_cert(b"foo", []) assert ca.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() def test_create_no_common_name(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca.get_cert(None, [])[0].cn is None def test_create_tmp(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca.get_cert(b"foo.com", []) assert ca.get_cert(b"foo.com", []) assert ca.get_cert(b"*.foo.com", []) @@ -57,7 +57,7 @@ class TestCertStore: assert r[1] == ca.default_privatekey def test_sans(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) c1 = ca.get_cert(b"foo.com", [b"*.bar.com"]) ca.get_cert(b"foo.bar.com", []) # assert c1 == c2 @@ -65,13 +65,13 @@ class TestCertStore: assert not c1 == c3 def test_sans_change(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) ca.get_cert(b"foo.com", [b"*.bar.com"]) cert, key, chain_file = ca.get_cert(b"foo.bar.com", [b"*.baz.com"]) assert b"*.baz.com" in cert.altnames def test_expire(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) ca.STORE_CAP = 3 ca.get_cert(b"one.com", []) ca.get_cert(b"two.com", []) @@ -95,8 +95,8 @@ class TestCertStore: assert (b"four.com", ()) in ca.certs def test_overrides(self, tmpdir): - ca1 = certs.CertStore.from_store(str(tmpdir.join("ca1")), "test") - ca2 = certs.CertStore.from_store(str(tmpdir.join("ca2")), "test") + ca1 = certs.CertStore.from_store(str(tmpdir.join("ca1")), "test", 2048) + ca2 = certs.CertStore.from_store(str(tmpdir.join("ca2")), "test", 2048) assert not ca1.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() dc = ca2.get_cert(b"foo.com", [b"sans.example.com"]) @@ -124,7 +124,7 @@ class TestCertStore: class TestDummyCert: def test_with_ca(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) r = certs.dummy_cert( ca.default_privatekey, ca.default_ca, -- cgit v1.2.3 From ed65476b2140435d857bd6acfffb5177ee71ad74 Mon Sep 17 00:00:00 2001 From: vin01 Date: Thu, 3 Oct 2019 10:23:45 +0200 Subject: Add EKU extension for dummy certificates --- mitmproxy/certs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 0ec63b50..65dc50e4 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -115,6 +115,13 @@ def dummy_cert(privkey, cacert, commonname, sans, organization): cert.set_version(2) cert.add_extensions( [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)]) + cert.add_extensions([ + OpenSSL.crypto.X509Extension( + b"extendedKeyUsage", + False, + b"serverAuth,clientAuth" + ) + ]) cert.set_pubkey(cacert.get_pubkey()) cert.sign(privkey, "sha256") return Cert(cert) -- cgit v1.2.3 From c88d3a943109132010a932aca82411a1527d934a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 6 Oct 2019 14:00:02 +0200 Subject: update dependencies --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 83981efb..3b9e1605 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ setup( "blinker>=1.4, <1.5", "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! - "click>=6.2, <7", + "click>=7.0,<8", "cryptography>=2.1.4,<2.5", "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", @@ -104,7 +104,7 @@ setup( "rstcheck>=2.2,<4.0", ], 'examples': [ - "beautifulsoup4>=4.4.1,<4.7" + "beautifulsoup4>=4.4.1,<4.9" ] } ) -- cgit v1.2.3 From 902ef59d01f45613ce33520159e157697bcc6f9f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 6 Oct 2019 14:41:46 +0200 Subject: Move onboardingapp from tornado to flask (#3661) --- mitmproxy/addons/onboarding.py | 3 +- mitmproxy/addons/onboardingapp/__init__.py | 37 +++++++ mitmproxy/addons/onboardingapp/app.py | 118 --------------------- .../addons/onboardingapp/static/mitmproxy.css | 4 +- .../addons/onboardingapp/templates/frame.html | 4 +- .../addons/onboardingapp/templates/index.html | 10 +- .../addons/onboardingapp/templates/layout.html | 2 +- mitmproxy/options.py | 1 + mitmproxy/proxy/config.py | 7 +- mitmproxy/tools/web/app.py | 15 ++- setup.cfg | 3 + setup.py | 3 +- test/mitmproxy/addons/onboardingapp/__init__.py | 0 test/mitmproxy/addons/onboardingapp/test_app.py | 1 - 14 files changed, 67 insertions(+), 141 deletions(-) delete mode 100644 mitmproxy/addons/onboardingapp/app.py delete mode 100644 test/mitmproxy/addons/onboardingapp/__init__.py delete mode 100644 test/mitmproxy/addons/onboardingapp/test_app.py diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 900acb08..94ca7c49 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -10,7 +10,7 @@ class Onboarding(wsgiapp.WSGIApp): name = "onboarding" def __init__(self): - super().__init__(app.Adapter(app.application), None, None) + super().__init__(app, None, None) def load(self, loader): loader.add_option( @@ -32,6 +32,7 @@ class Onboarding(wsgiapp.WSGIApp): def configure(self, updated): self.host = ctx.options.onboarding_host self.port = ctx.options.onboarding_port + app.config["CONFDIR"] = ctx.options.confdir def request(self, f): if ctx.options.onboarding: diff --git a/mitmproxy/addons/onboardingapp/__init__.py b/mitmproxy/addons/onboardingapp/__init__.py index e69de29b..722fed03 100644 --- a/mitmproxy/addons/onboardingapp/__init__.py +++ b/mitmproxy/addons/onboardingapp/__init__.py @@ -0,0 +1,37 @@ +import os + +from flask import Flask, render_template + +from mitmproxy.options import CONF_BASENAME, CONF_DIR + +app = Flask(__name__) +# will be overridden in the addon, setting this here so that the Flask app can be run standalone. +app.config["CONFDIR"] = CONF_DIR + + +@app.route('/') +def index(): + return render_template("index.html") + + +@app.route('/cert/pem') +def pem(): + return read_cert("pem", "application/x-x509-ca-cert") + + +@app.route('/cert/p12') +def p12(): + return read_cert("p12", "application/x-pkcs12") + + +def read_cert(ext, content_type): + filename = CONF_BASENAME + f"-ca-cert.{ext}" + p = os.path.join(app.config["CONFDIR"], filename) + p = os.path.expanduser(p) + with open(p, "rb") as f: + cert = f.read() + + return cert, { + "Content-Type": content_type, + "Content-Disposition": f"inline; filename={filename}", + } diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py deleted file mode 100644 index ab136778..00000000 --- a/mitmproxy/addons/onboardingapp/app.py +++ /dev/null @@ -1,118 +0,0 @@ -import os - -import tornado.template -import tornado.web -import tornado.wsgi - -from mitmproxy.utils import data -from mitmproxy.proxy import config - -loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates")) - - -class Adapter(tornado.wsgi.WSGIAdapter): - # Tornado doesn't make the WSGI environment available to pages, so this - # hideous monkey patch is the easiest way to get to the mitmproxy.master - # variable. - - def __init__(self, application): - self._application = application - - def application(self, request): - request.master = self.environ["mitmproxy.master"] - return self._application(request) - - def __call__(self, environ, start_response): - self.environ = environ - return tornado.wsgi.WSGIAdapter.__call__( - self, - environ, - start_response - ) - - -class Index(tornado.web.RequestHandler): - - def get(self): - t = loader.load("index.html") - self.write(t.generate()) - - -class PEM(tornado.web.RequestHandler): - - @property - def filename(self): - return config.CONF_BASENAME + "-ca-cert.pem" - - def head(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - content_length = os.path.getsize(p) - - self.set_header("Content-Type", "application/x-x509-ca-cert") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - self.set_header("Content-Length", content_length) - - def get(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - self.set_header("Content-Type", "application/x-x509-ca-cert") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - with open(p, "rb") as f: - self.write(f.read()) - - -class P12(tornado.web.RequestHandler): - - @property - def filename(self): - return config.CONF_BASENAME + "-ca-cert.p12" - - def head(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - content_length = os.path.getsize(p) - - self.set_header("Content-Type", "application/x-pkcs12") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - self.set_header("Content-Length", content_length) - - def get(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - self.set_header("Content-Type", "application/x-pkcs12") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - with open(p, "rb") as f: - self.write(f.read()) - - -application = tornado.web.Application( - [ - (r"/", Index), - (r"/cert/pem", PEM), - (r"/cert/p12", P12), - ( - r"/static/(.*)", - tornado.web.StaticFileHandler, - { - "path": data.pkg_data.path("addons/onboardingapp/static") - } - ), - ], - # debug=True -) diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css index 969bd62b..e654d56b 100644 --- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css +++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css @@ -15,7 +15,7 @@ height: 300px; } -.bigtitle>div { +.bigtitle > div { display: table-cell; vertical-align: middle; } @@ -31,7 +31,7 @@ section { .innerlink { text-decoration: none; - border-bottom:1px dotted; + border-bottom: 1px dotted; margin-bottom: 15px; } diff --git a/mitmproxy/addons/onboardingapp/templates/frame.html b/mitmproxy/addons/onboardingapp/templates/frame.html index f00e1a66..13003f3c 100644 --- a/mitmproxy/addons/onboardingapp/templates/frame.html +++ b/mitmproxy/addons/onboardingapp/templates/frame.html @@ -3,7 +3,7 @@
{% block body %} - {% end %} + {% endblock %}
-{% end %} +{% endblock %} diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index 38aa27ed..aa471668 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -135,19 +135,19 @@ function changeTo(device) {

Click to install your mitmproxy certificate

- +

Apple

- +

Windows

- +

Android

- +

Other

@@ -167,4 +167,4 @@ function changeTo(device) { between mitmproxy installations. -{% end %} +{% endblock %} diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html index f6e1b286..cea8373b 100644 --- a/mitmproxy/addons/onboardingapp/templates/layout.html +++ b/mitmproxy/addons/onboardingapp/templates/layout.html @@ -28,7 +28,7 @@
{% block content %} - {% end %} + {% endblock %}
diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 69ffd033..1583e9fc 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -5,6 +5,7 @@ from mitmproxy.net import tls CONF_DIR = "~/.mitmproxy" +CONF_BASENAME = "mitmproxy" LISTEN_PORT = 8080 CONTENT_VIEW_LINES_CUTOFF = 512 KEY_SIZE = 2048 diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 881ed2fd..ae2ec68b 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -4,16 +4,13 @@ import typing from OpenSSL import crypto +from mitmproxy import certs from mitmproxy import exceptions from mitmproxy import options as moptions -from mitmproxy import certs from mitmproxy.net import server_spec -CONF_BASENAME = "mitmproxy" - class HostMatcher: - def __init__(self, handle, patterns=tuple()): self.handle = handle self.patterns = list(patterns) @@ -67,7 +64,7 @@ class ProxyConfig: key_size = options.key_size self.certstore = certs.CertStore.from_store( certstore_path, - CONF_BASENAME, + moptions.CONF_BASENAME, key_size ) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 6bfce34e..acf2cfdf 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -1,24 +1,25 @@ +import asyncio import hashlib import json import logging import os.path import re from io import BytesIO -import asyncio -import mitmproxy.flow import tornado.escape import tornado.web import tornado.websocket + +import mitmproxy.flow +import mitmproxy.tools.web.master # noqa from mitmproxy import contentviews from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io from mitmproxy import log -from mitmproxy import version from mitmproxy import optmanager -import mitmproxy.tools.web.master # noqa +from mitmproxy import version def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: @@ -108,6 +109,8 @@ class APIError(tornado.web.HTTPError): class RequestHandler(tornado.web.RequestHandler): + application: "Application" + def write(self, chunk): # Writing arrays on the top level is ok nowadays. # http://flask.pocoo.org/docs/0.11/security/#json-security @@ -473,7 +476,9 @@ class DnsRebind(RequestHandler): class Application(tornado.web.Application): - def __init__(self, master, debug): + master: "mitmproxy.tools.web.master.WebMaster" + + def __init__(self, master: "mitmproxy.tools.web.master.WebMaster", debug: bool) -> None: self.master = master super().__init__( default_host="dns-rebind-protection", diff --git a/setup.cfg b/setup.cfg index c717bd1c..0a34b805 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,9 @@ exclude_lines = [mypy-mitmproxy.contrib.*] ignore_errors = True +[mypy-tornado.*] +ignore_errors = True + [tool:full_coverage] exclude = mitmproxy/proxy/protocol/base.py diff --git a/setup.py b/setup.py index 3b9e1605..7c5f1eb2 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ setup( "certifi>=2019.9.11", # no semver here - this should always be on the last release! "click>=7.0,<8", "cryptography>=2.1.4,<2.5", + "flask>=1.1.1,<1.2", "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", @@ -78,7 +79,7 @@ setup( "pyperclip>=1.6.0,<1.8", "ruamel.yaml>=0.16,<0.17", "sortedcontainers>=2.1.0,<2.2", - "tornado>=4.3,<5.2", + "tornado>=4.3,<7", "urwid>=2.0.1,<2.1", "wsproto>=0.14.0,<0.15.0", "publicsuffix2>=2.20190812,<3", diff --git a/test/mitmproxy/addons/onboardingapp/__init__.py b/test/mitmproxy/addons/onboardingapp/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/mitmproxy/addons/onboardingapp/test_app.py b/test/mitmproxy/addons/onboardingapp/test_app.py deleted file mode 100644 index 777ab4dd..00000000 --- a/test/mitmproxy/addons/onboardingapp/test_app.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests -- cgit v1.2.3 From f580e0ea97a33733434cdcfd9c712140307b7439 Mon Sep 17 00:00:00 2001 From: jannst Date: Sun, 13 Oct 2019 17:48:23 +0200 Subject: hugo server -D fails if html file in docs/src/generated are not present --- docs/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/README.md b/docs/README.md index cc06f081..b39fd780 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,9 @@ This directory houses the mitmproxy documentation available at Date: Sun, 13 Oct 2019 18:20:38 +0200 Subject: improve wording --- docs/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index b39fd780..a9ee1113 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,10 +7,8 @@ This directory houses the mitmproxy documentation available at Date: Sun, 13 Oct 2019 22:03:24 +0200 Subject: Adding tutorial on how to insert mitmproxy CA cert into the android system certificate store --- .../howto-install-system-trusted-ca-android.md | 86 ++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 docs/src/content/howto-install-system-trusted-ca-android.md diff --git a/docs/src/content/howto-install-system-trusted-ca-android.md b/docs/src/content/howto-install-system-trusted-ca-android.md new file mode 100644 index 00000000..2ef67f30 --- /dev/null +++ b/docs/src/content/howto-install-system-trusted-ca-android.md @@ -0,0 +1,86 @@ +--- +title: "Install System CA on Android" +menu: + howto: + weight: 4 +--- + +# Install System CA Certificate on Android Emulator + +[Since Android 7, apps ignore user certificates](https://android-developers.googleblog.com/2016/07/changes-to-trusted-certificate.html), unless they are configured to use them. +As most applications do not explicitly opt in to use user certificates, we need to place our mitmproxy CA certificate in the system certificate store, +in order to avid having to patch each application, which we want to monitor. + +Please note, that apps can decide to ignore the system certificate store and maintain their own CA certificates. In this case you have to patch the application. + +## 1. Prerequisites + + - Emulator from Android SDK with proxy settings pointing to mitmproxy + + - Mitmproxy CA certificate + - Usually located in `~/.mitmproxy/mitmproxy-ca-cert.cer` + - If the folder is empty or does not exist, run `mitmproxy` in order to generate the certificates + +## 2. Rename certificate +Enter your certificate folder +{{< highlight bash >}} +cd ~/.mitmproxy/ +{{< / highlight >}} + + - CA Certificates in Android are stored by the name of their hash, with a '0' as extension + - Now generate the hash of your certificate + +{{< highlight bash >}} +openssl x509 -inform PEM -subject_hash_old -in mitmproxy-ca-cert.cer | head -1 +{{< / highlight >}} +Lets assume, the output is `c8450d0d` + +We can now copy `mitmproxy-ca-cert.cer` to `c8450d0d.0` and our system certificate is ready to use +{{< highlight bash >}} +cp mitmproxy-ca-cert.cer c8450d0d.0 +{{< / highlight >}} + +## 3. Insert certificate into system certificate store + +Note, that Android 9 (API LEVEL 28) was used to test the following steps and that the `emulator` executable is located in the Android SDK + + - Start your android emulator. + - Get a list of your AVDs with `emulator -list-avds` + - Make sure to use the `-writable-system` option. Otherwise it will not be possible to write to `/system` + - Keep in mind, that the **emulator will load a clean system image when starting without `-writable-system` option**. + - This means you always have to start the emulator with `-writable-system` option in order to use your certificate + +{{< highlight bash >}} +emulator -avd -writable-system +{{< / highlight >}} + + - Restart adb as root + +{{< highlight bash >}} +adb root +{{< / highlight >}} + + - Get write access to `/system` on the device + - In earlier versions (API LEVEL < 28) of Android you have to use `adb shell "mount -o rw,remount /system"` + +{{< highlight bash >}} +adb shell "mount -o rw,remount /" +{{< / highlight >}} + + - Push your certificate to the system certificate store and set file permissions + +{{< highlight bash >}} +adb push c8450d0d.0 /system/etc/security/cacerts +adb shell "chmod 664 /system/etc/security/cacerts/c8450d0d.0" +{{< / highlight >}} + +## 4. Reboot device and enjoy decrypted TLS traffic + + - Reboot your device. + - You CA certificate should now be system trusted + +{{< highlight bash >}} +adb reboot +{{< / highlight >}} + +**Remember**: You **always** have to start the emulator using the `-writable-system` option in order to use your certificate \ No newline at end of file -- cgit v1.2.3 From 45e3ae0f9cb50b0edbf4180fd969ea99d40bdf7b Mon Sep 17 00:00:00 2001 From: Will Date: Tue, 15 Oct 2019 11:59:42 -0600 Subject: grammer correction (#3670) --- docs/src/content/howto-ignoredomains.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/howto-ignoredomains.md b/docs/src/content/howto-ignoredomains.md index 902a17be..9a337eba 100644 --- a/docs/src/content/howto-ignoredomains.md +++ b/docs/src/content/howto-ignoredomains.md @@ -10,7 +10,7 @@ menu: There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism: -- **Certificate pinning:** Some traffic is is protected using [Certificate +- **Certificate pinning:** Some traffic is protected using [Certificate Pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's interception leads to errors. For example, the Twitter app, Windows Update or the Apple App Store fail to work if mitmproxy is active. -- cgit v1.2.3 From 063ff41858623eec9da59e40245afbe7b86772a4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 16 Oct 2019 20:33:09 +0200 Subject: add http message type hints --- mitmproxy/net/http/message.py | 24 ++++++++++++++---------- mitmproxy/net/http/request.py | 2 ++ mitmproxy/net/http/response.py | 2 ++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 6830c6cd..d5a7ff9c 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -1,14 +1,18 @@ import re -from typing import Optional, Union # noqa +from typing import Optional # noqa from mitmproxy.utils import strutils from mitmproxy.net.http import encoding from mitmproxy.coretypes import serializable -from mitmproxy.net.http import headers +from mitmproxy.net.http import headers as mheaders class MessageData(serializable.Serializable): - content: bytes = None + headers: mheaders.Headers + content: bytes + http_version: bytes + timestamp_start: float + timestamp_end: float def __eq__(self, other): if isinstance(other, MessageData): @@ -18,7 +22,7 @@ class MessageData(serializable.Serializable): def set_state(self, state): for k, v in state.items(): if k == "headers": - v = headers.Headers.from_state(v) + v = mheaders.Headers.from_state(v) setattr(self, k, v) def get_state(self): @@ -28,12 +32,12 @@ class MessageData(serializable.Serializable): @classmethod def from_state(cls, state): - state["headers"] = headers.Headers.from_state(state["headers"]) + state["headers"] = mheaders.Headers.from_state(state["headers"]) return cls(**state) class Message(serializable.Serializable): - data: MessageData = None + data: MessageData def __eq__(self, other): if isinstance(other, Message): @@ -48,7 +52,7 @@ class Message(serializable.Serializable): @classmethod def from_state(cls, state): - state["headers"] = headers.Headers.from_state(state["headers"]) + state["headers"] = mheaders.Headers.from_state(state["headers"]) return cls(**state) @property @@ -160,7 +164,7 @@ class Message(serializable.Serializable): self.data.timestamp_end = timestamp_end def _get_content_type_charset(self) -> Optional[str]: - ct = headers.parse_content_type(self.headers.get("content-type", "")) + ct = mheaders.parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") return None @@ -213,9 +217,9 @@ class Message(serializable.Serializable): self.content = encoding.encode(text, enc) except ValueError: # Fall back to UTF-8 and update the content-type header. - ct = headers.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) + ct = mheaders.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) ct[2]["charset"] = "utf-8" - self.headers["content-type"] = headers.assemble_content_type(*ct) + self.headers["content-type"] = mheaders.assemble_content_type(*ct) enc = "utf8" self.content = text.encode(enc, "surrogateescape") diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index ef33ca49..1569ea72 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -64,6 +64,8 @@ class Request(message.Message): """ An HTTP request. """ + data: RequestData + def __init__(self, *args, **kwargs): super().__init__() self.data = RequestData(*args, **kwargs) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 9491fc03..2e864405 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -47,6 +47,8 @@ class Response(message.Message): """ An HTTP response. """ + data: ResponseData + def __init__(self, *args, **kwargs): super().__init__() self.data = ResponseData(*args, **kwargs) -- cgit v1.2.3 From ad120c380aeca740d3a9a24b05dc3f181a9557b7 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Fri, 25 Oct 2019 10:38:35 +0200 Subject: fixes #3559 if the `:authority` header is not found, we can try to guess it from variable `flow.request.pretty_host` as .pop() accepts a fallback parameter --- mitmproxy/addons/clientplayback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index c56c0e74..13fbfebb 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -203,7 +203,7 @@ class ClientPlayback: # https://github.com/mitmproxy/mitmproxy/issues/2197 if hf.request.http_version == "HTTP/2.0": hf.request.http_version = "HTTP/1.1" - host = hf.request.headers.pop(":authority") + host = hf.request.headers.pop(":authority", hf.request.pretty_host) hf.request.headers.insert(0, "host", host) self.q.put(hf) ctx.master.addons.trigger("update", lst) -- cgit v1.2.3 From 33707403614642ae63d7ed24f36c80a3f58ce7ac Mon Sep 17 00:00:00 2001 From: Yoann L Date: Mon, 28 Oct 2019 17:51:59 +0100 Subject: several fixes on command exports has several problems: #3676 * authority can usually rely on actual URL. as `:authority` headers will break curl command. (advise if it's better to change them to Host, or if it should be reported on curl side) * `content-length`: 0 is added for each request. if it's found in the curl argument list, it'll try to fetch an empty body (and crash). also trying to guess on accept-encoding header to add the `--compress` option when fetching potentially compressed content. * ditto for httpie --- mitmproxy/addons/export.py | 15 +++++++++++++++ test/mitmproxy/addons/test_export.py | 24 +++++++++--------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 90e95d3e..0c10f352 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -11,6 +11,15 @@ import mitmproxy.types import pyperclip +def cleanup_for_commands(request): + if ('content-length' in request.headers.keys() and + request.headers['content-length'] == '0' and + request.method == 'GET'): + request.headers.pop('content-length') + request.headers.pop(':authority', None) + return request + + def raise_if_missing_request(f: flow.Flow) -> None: if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") @@ -21,10 +30,15 @@ def curl_command(f: flow.Flow) -> str: data = "curl " request = f.request.copy() # type: ignore request.decode(strict=False) + request = cleanup_for_commands(request) for k, v in request.headers.items(multi=True): data += "-H '%s:%s' " % (k, v) if request.method != "GET": data += "-X %s " % request.method + for header in request.headers.keys(): + if ('accept-encoding' == header.lower() and + any(comp in request.headers[header] for comp in ['gzip', 'deflate'])): + data += "--compressed " data += "'%s'" % request.url if request.content: data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( @@ -39,6 +53,7 @@ def httpie_command(f: flow.Flow) -> str: request = f.request.copy() # type: ignore data = "http %s " % request.method request.decode(strict=False) + request = cleanup_for_commands(request) data += "%s" % request.url for k, v in request.headers.items(multi=True): data += " '%s:%s'" % (k, v) diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index f4bb0f64..c86e0c7d 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -14,29 +14,23 @@ from unittest import mock @pytest.fixture def get_request(): return tflow.tflow( - req=tutils.treq( - method=b'GET', - content=b'', - path=b"/path?a=foo&a=bar&b=baz" - ) - ) + req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz")) @pytest.fixture def post_request(): return tflow.tflow( - req=tutils.treq( - method=b'POST', - headers=(), - content=bytes(range(256)) - ) - ) + req=tutils.treq(method=b'POST', headers=(), content=bytes(range(256)))) @pytest.fixture def patch_request(): return tflow.tflow( - req=tutils.treq(method=b'PATCH', path=b"/path?query=param") + req=tutils.treq( + method=b'PATCH', + content=b'content', + path=b"/path?query=param" + ) ) @@ -47,7 +41,7 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H 'header:qvalue' -H 'content-length:0' 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H 'header:qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): @@ -67,7 +61,7 @@ class TestExportCurlCommand: class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue' 'content-length:0'""" + result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue'""" assert export.httpie_command(get_request) == result def test_post(self, post_request): -- cgit v1.2.3 From 4383122b7b6930612ba865c6d027eb375ecce058 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Tue, 29 Oct 2019 12:24:07 +0100 Subject: several fixes on command exports has several problems: #3676 * authority can usually rely on actual URL. as `:authority` headers will break curl command. (advise if it's better to change them to Host, or if it should be reported on curl side) * `content-length`: 0 is added for each request. if it's found in the curl argument list, it'll try to fetch an empty body (and crash). also trying to guess on accept-encoding header to add the `--compress` option when fetching potentially compressed content. * ditto for httpie * factorize it into raise_if_missing_request (and rename it) --- mitmproxy/addons/export.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 0c10f352..744db0fc 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -11,7 +11,12 @@ import mitmproxy.types import pyperclip -def cleanup_for_commands(request): +def cleanup_request(f: flow.Flow): + if not hasattr(f, "request"): + raise exceptions.CommandError("Can't export flow with no request.") + request = f.request.copy() + request.decode(strict=False) + # a bit of clean-up if ('content-length' in request.headers.keys() and request.headers['content-length'] == '0' and request.method == 'GET'): @@ -20,25 +25,14 @@ def cleanup_for_commands(request): return request -def raise_if_missing_request(f: flow.Flow) -> None: - if not hasattr(f, "request"): - raise exceptions.CommandError("Can't export flow with no request.") - - def curl_command(f: flow.Flow) -> str: - raise_if_missing_request(f) data = "curl " - request = f.request.copy() # type: ignore - request.decode(strict=False) - request = cleanup_for_commands(request) + request = cleanup_request(f) for k, v in request.headers.items(multi=True): + data += "--compressed " if k == 'accept-encoding' else "" data += "-H '%s:%s' " % (k, v) if request.method != "GET": data += "-X %s " % request.method - for header in request.headers.keys(): - if ('accept-encoding' == header.lower() and - any(comp in request.headers[header] for comp in ['gzip', 'deflate'])): - data += "--compressed " data += "'%s'" % request.url if request.content: data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( @@ -49,12 +43,8 @@ def curl_command(f: flow.Flow) -> str: def httpie_command(f: flow.Flow) -> str: - raise_if_missing_request(f) - request = f.request.copy() # type: ignore - data = "http %s " % request.method - request.decode(strict=False) - request = cleanup_for_commands(request) - data += "%s" % request.url + request = cleanup_request(f) + data = "http %s %s" % (request.method, request.url) for k, v in request.headers.items(multi=True): data += " '%s:%s'" % (k, v) if request.content: @@ -66,8 +56,7 @@ def httpie_command(f: flow.Flow) -> str: def raw(f: flow.Flow) -> bytes: - raise_if_missing_request(f) - return assemble.assemble_request(f.request) # type: ignore + return assemble.assemble_request(cleanup_request(f)) # type: ignore formats = dict( -- cgit v1.2.3 From 6da1db750c7fc105d1d312fe6520e1b1f9d32567 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Tue, 29 Oct 2019 12:38:11 +0100 Subject: missed a type ignore, for mypy compliance --- mitmproxy/addons/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 744db0fc..cc698f9c 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -14,7 +14,7 @@ import pyperclip def cleanup_request(f: flow.Flow): if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") - request = f.request.copy() + request = f.request.copy() # type: ignore request.decode(strict=False) # a bit of clean-up if ('content-length' in request.headers.keys() and -- cgit v1.2.3 From fcccab684d61d9c428fd9b862db14dd9da5a0e38 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 29 Oct 2019 16:15:07 +0100 Subject: simplify condition --- mitmproxy/addons/export.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index cc698f9c..2776118a 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -17,9 +17,7 @@ def cleanup_request(f: flow.Flow): request = f.request.copy() # type: ignore request.decode(strict=False) # a bit of clean-up - if ('content-length' in request.headers.keys() and - request.headers['content-length'] == '0' and - request.method == 'GET'): + if request.method == 'GET' and request.headers.get("content-length", None) == "0": request.headers.pop('content-length') request.headers.pop(':authority', None) return request -- cgit v1.2.3 From 1469678b99dcdbd8af5d899f6a8b1036e03ce70f Mon Sep 17 00:00:00 2001 From: Yoann L Date: Mon, 4 Nov 2019 12:26:21 +0100 Subject: take no action if on `Host` if `:authority` isn't found --- mitmproxy/addons/clientplayback.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 13fbfebb..7bdaeb33 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -203,8 +203,9 @@ class ClientPlayback: # https://github.com/mitmproxy/mitmproxy/issues/2197 if hf.request.http_version == "HTTP/2.0": hf.request.http_version = "HTTP/1.1" - host = hf.request.headers.pop(":authority", hf.request.pretty_host) - hf.request.headers.insert(0, "host", host) + host = hf.request.headers.pop(":authority", None) + if host is not None: + hf.request.headers.insert(0, "host", host) self.q.put(hf) ctx.master.addons.trigger("update", lst) -- cgit v1.2.3 From 9c9cfdd7ec10eb00c75c56c6ec62bea7f923af62 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 4 Nov 2019 15:43:39 +0100 Subject: Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 19 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..01b6fb85 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,19 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: kind/triage +assignees: '' + +--- + +#### Problem Description +A clear and concise description of what the bug is. + +#### Steps to reproduce the behavior: +1. +2. +3. + +#### System Information +Paste the output of "mitmproxy --version" here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..8e8080db --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: kind/feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +#### Describe the solution you'd like +A clear and concise description of what you want to happen. + +#### Describe alternatives you've considered +A clear and concise description of any alternative solutions or features you've considered. + +#### Additional context +Add any other context or screenshots about the feature request here. -- cgit v1.2.3 From 3fc648a6d2c552370e3fc0d42c0d9414e9595b1c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 4 Nov 2019 15:45:18 +0100 Subject: Update issue_template.md --- issue_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_template.md b/issue_template.md index 2ea213a5..3dbac2ac 100644 --- a/issue_template.md +++ b/issue_template.md @@ -14,4 +14,4 @@ - + -- cgit v1.2.3 From 0a197af6a2d337bbafed34ce6718caa6b4139034 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 4 Nov 2019 16:03:04 +0100 Subject: update badges --- README.rst | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index d8938a3e..99797d19 100644 --- a/README.rst +++ b/README.rst @@ -30,12 +30,10 @@ The documentation for mitmproxy is available on our website: |mitmproxy_docs_stable| |mitmproxy_docs_master| +If you have questions on how to use mitmproxy, please +ask them on StackOverflow! -Join our discussion forum on Discourse to ask questions, help -each other solve problems, and come up with new ideas for the project. - -|mitmproxy_discourse| - +|mitmproxy_stackoverflow| Join our developer chat on Slack if you would like to contribute to mitmproxy itself. @@ -146,21 +144,21 @@ with the following command: tox -e lint -.. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg +.. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg :target: https://mitmproxy.org/ :alt: mitmproxy.org -.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/api/docs-stable-brightgreen.svg +.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/badge/docs-stable-brightgreen.svg :target: https://docs.mitmproxy.org/stable/ :alt: mitmproxy documentation stable -.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/api/docs-master-brightgreen.svg +.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/badge/docs-master-brightgreen.svg :target: https://docs.mitmproxy.org/master/ :alt: mitmproxy documentation master -.. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg - :target: https://discourse.mitmproxy.org - :alt: Discourse: mitmproxy +.. |mitmproxy_stackoverflow| image:: https://shields.mitmproxy.org/stackexchange/stackoverflow/t/mitmproxy?color=orange&label=stackoverflow%20questions + :target: https://stackoverflow.com/questions/tagged/mitmproxy + :alt: StackOverflow: mitmproxy .. |slack| image:: http://slack.mitmproxy.org/badge.svg :target: http://slack.mitmproxy.org/ -- cgit v1.2.3 From 3af4647804700bb6e86a9e1b73d7bf8612d872fa Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 4 Nov 2019 16:05:43 +0100 Subject: squash remaining forum reference --- README.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 99797d19..e41dc80b 100644 --- a/README.rst +++ b/README.rst @@ -21,8 +21,7 @@ Documentation & Help -------------------- -General information, tutorials, and precompiled binaries can be found on the mitmproxy -and pathod websites. +General information, tutorials, and precompiled binaries can be found on the mitmproxy website. |mitmproxy_site| @@ -52,7 +51,7 @@ Contributing As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward, please consider contributing in the following areas: -- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. +- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users on StackOverflow_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. - **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first. Development Setup @@ -193,5 +192,5 @@ with the following command: .. _yarn: https://yarnpkg.com/en/ .. _PEP8: https://www.python.org/dev/peps/pep-0008 .. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html -.. _forums: https://discourse.mitmproxy.org/ +.. _StackOverflow: https://stackoverflow.com/questions/tagged/mitmproxy .. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 -- cgit v1.2.3 From d35c00ee65708ca73005e201fa1998a628f553b3 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 28 Oct 2019 16:23:15 +0100 Subject: Added support for IPv6 in pf.py for macOS --- mitmproxy/platform/pf.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py index 5e22ec31..fd62d977 100644 --- a/mitmproxy/platform/pf.py +++ b/mitmproxy/platform/pf.py @@ -13,9 +13,15 @@ def lookup(address, port, s): # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) s = s.decode() - spec = "%s:%s" % (address, port) + + # ALL tcp 192.168.1.13:57474 -> 23.205.82.58:443 ESTABLISHED:ESTABLISHED + specv4 = "%s:%s" % (address, port) + + # ALL tcp 2a01:e35:8bae:50f0:9d9b:ef0d:2de3:b733[58505] -> 2606:4700:30::681f:4ad0[443] ESTABLISHED:ESTABLISHED + specv6 = "%s[%s]" % (address, port) + for i in s.split("\n"): - if "ESTABLISHED:ESTABLISHED" in i and spec in i: + if "ESTABLISHED:ESTABLISHED" in i and specv4 in i: s = i.split() if len(s) > 4: if sys.platform.startswith("freebsd"): @@ -26,4 +32,11 @@ def lookup(address, port, s): if len(s) == 2: return s[0], int(s[1]) + elif "ESTABLISHED:ESTABLISHED" in i and specv6 in i: + s = i.split() + if len(s) > 4: + s = s[4].split("[") + port = s[1].split("]") + port = port[0] + return s[0], int(port) raise RuntimeError("Could not resolve original destination.") -- cgit v1.2.3 From 93c103a2a2263311dcc006cbcddbd84aa50ac052 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 31 Oct 2019 18:40:33 +0100 Subject: Fixed lint --- mitmproxy/platform/pf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py index fd62d977..74e077a4 100644 --- a/mitmproxy/platform/pf.py +++ b/mitmproxy/platform/pf.py @@ -14,12 +14,12 @@ def lookup(address, port, s): address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) s = s.decode() - # ALL tcp 192.168.1.13:57474 -> 23.205.82.58:443 ESTABLISHED:ESTABLISHED + # ALL tcp 192.168.1.13:57474 -> 23.205.82.58:443 ESTABLISHED:ESTABLISHED specv4 = "%s:%s" % (address, port) - - # ALL tcp 2a01:e35:8bae:50f0:9d9b:ef0d:2de3:b733[58505] -> 2606:4700:30::681f:4ad0[443] ESTABLISHED:ESTABLISHED + + # ALL tcp 2a01:e35:8bae:50f0:9d9b:ef0d:2de3:b733[58505] -> 2606:4700:30::681f:4ad0[443] ESTABLISHED:ESTABLISHED specv6 = "%s[%s]" % (address, port) - + for i in s.split("\n"): if "ESTABLISHED:ESTABLISHED" in i and specv4 in i: s = i.split() -- cgit v1.2.3 From ff628e783e8560da50b4dbfac33906572c66f3ad Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 31 Oct 2019 19:33:13 +0100 Subject: pfctl state output always have 2 lines for each socket. Adding outgoing lines in data which matches lines before incoming ones. Also adding IPv6 data and tests --- test/mitmproxy/data/pf01 | 6 ++++++ test/mitmproxy/platform/test_pf.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/test/mitmproxy/data/pf01 b/test/mitmproxy/data/pf01 index 3139a289..019a6b76 100644 --- a/test/mitmproxy/data/pf01 +++ b/test/mitmproxy/data/pf01 @@ -1,4 +1,10 @@ No ALTQ support in kernel ALTQ related functions disabled +ALL tcp 192.168.1.111:40001 -> 5.5.5.6:80 FIN_WAIT_2:FIN_WAIT_2 ALL tcp 127.0.0.1:8080 <- 5.5.5.6:80 <- 192.168.1.111:40001 FIN_WAIT_2:FIN_WAIT_2 +ALL tcp 192.168.1.111:40000 -> 5.5.5.5:80 ESTABLISHED:ESTABLISHED ALL tcp 127.0.0.1:8080 <- 5.5.5.5:80 <- 192.168.1.111:40000 ESTABLISHED:ESTABLISHED +ALL tcp 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40002] -> 2a03:2880:f21f:c5:face:b00c::167[443] ESTABLISHED:ESTABLISHED +ALL tcp ::1[8080] <- 2a03:2880:f21f:c5:face:b00c::167[443] <- 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40002] ESTABLISHED:ESTABLISHED +ALL tcp 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40003] -> 2a03:2880:f21f:c5:face:b00c::167[443] FIN_WAIT_2:FIN_WAIT_2 +ALL tcp ::1[6970] <- 2a03:2880:f21f:c5:face:b00c::167[443] <- 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40003] FIN_WAIT_2:FIN_WAIT_2 \ No newline at end of file diff --git a/test/mitmproxy/platform/test_pf.py b/test/mitmproxy/platform/test_pf.py index 9795a2db..4a7dfe75 100644 --- a/test/mitmproxy/platform/test_pf.py +++ b/test/mitmproxy/platform/test_pf.py @@ -19,3 +19,8 @@ class TestLookup: pf.lookup("192.168.1.112", 40000, d) with pytest.raises(Exception, match="Could not resolve original destination"): pf.lookup("192.168.1.111", 40001, d) + assert pf.lookup("2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db", 40002, d) == ("2a03:2880:f21f:c5:face:b00c::167", 443) + with pytest.raises(Exception, match="Could not resolve original destination"): + pf.lookup("2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db", 40003, d) + with pytest.raises(Exception, match="Could not resolve original destination"): + pf.lookup("2a01:e35:face:face:face:face:face:face", 40003, d) -- cgit v1.2.3 From 80963966b267a4c3723e4773774b854602247bca Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Nov 2019 18:19:50 +0100 Subject: make duration formatting more forgiving --- mitmproxy/utils/human.py | 4 +++- test/mitmproxy/utils/test_human.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index 5c02b072..3158a294 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -48,12 +48,14 @@ def parse_size(s: typing.Optional[str]) -> typing.Optional[int]: raise ValueError("Invalid size specification.") -def pretty_duration(secs): +def pretty_duration(secs: typing.Optional[float]) -> str: formatters = [ (100, "{:.0f}s"), (10, "{:2.1f}s"), (1, "{:1.2f}s"), ] + if secs is None: + return "" for limit, formatter in formatters: if secs >= limit: diff --git a/test/mitmproxy/utils/test_human.py b/test/mitmproxy/utils/test_human.py index faf35f72..6f8bf732 100644 --- a/test/mitmproxy/utils/test_human.py +++ b/test/mitmproxy/utils/test_human.py @@ -47,6 +47,7 @@ def test_pretty_duration(): assert human.pretty_duration(10000) == "10000s" assert human.pretty_duration(1.123) == "1.12s" assert human.pretty_duration(0.123) == "123ms" + assert human.pretty_duration(None) == "" def test_format_address(): -- cgit v1.2.3 From f97996126f7a7606f8601f0318f0a70a4e818c6c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 11 Nov 2019 18:35:06 +0100 Subject: minor improvements and sans-io adjustments --- mitmproxy/certs.py | 9 +++++++-- mitmproxy/net/tls.py | 28 ++++++++++++++-------------- mitmproxy/proxy/protocol/tls.py | 25 +++++++++++-------------- mitmproxy/proxy/root_context.py | 2 +- test/mitmproxy/net/test_tls.py | 2 +- test/mitmproxy/test_connections.py | 2 +- 6 files changed, 35 insertions(+), 33 deletions(-) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 65dc50e4..e702e1cf 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -123,7 +123,7 @@ def dummy_cert(privkey, cacert, commonname, sans, organization): ) ]) cert.set_pubkey(cacert.get_pubkey()) - cert.sign(privkey, "sha256") + cert.sign(privkey, b"sha256") return Cert(cert) @@ -315,7 +315,12 @@ class CertStore: ret.append(b"*." + b".".join(parts[i:])) return ret - def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes], organization: typing.Optional[bytes] = None): + def get_cert( + self, + commonname: typing.Optional[bytes], + sans: typing.List[bytes], + organization: typing.Optional[bytes] = None + ) -> typing.Tuple["Cert", OpenSSL.SSL.PKey, str]: """ Returns an (cert, privkey, cert_chain) tuple. diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 4dc61969..48392d1b 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -295,6 +295,17 @@ def create_client_context( return context +def accept_all( + conn_: SSL.Connection, + x509: SSL.X509, + errno: int, + err_depth: int, + is_cert_verified: bool, +) -> bool: + # Return true to prevent cert verification error + return True + + def create_server_context( cert: typing.Union[certs.Cert, str], key: SSL.PKey, @@ -324,16 +335,6 @@ def create_server_context( until then we're conservative. """ - def accept_all( - conn_: SSL.Connection, - x509: SSL.X509, - errno: int, - err_depth: int, - is_cert_verified: bool, - ) -> bool: - # Return true to prevent cert verification error - return True - if request_client_cert: verify = SSL.VERIFY_PEER else: @@ -425,7 +426,7 @@ class ClientHello: return self._client_hello.cipher_suites.cipher_suites @property - def sni(self): + def sni(self) -> typing.Optional[bytes]: if self._client_hello.extensions: for extension in self._client_hello.extensions.extensions: is_valid_sni_extension = ( @@ -435,7 +436,7 @@ class ClientHello: check.is_valid_host(extension.body.server_names[0].host_name) ) if is_valid_sni_extension: - return extension.body.server_names[0].host_name.decode("idna") + return extension.body.server_names[0].host_name return None @property @@ -478,5 +479,4 @@ class ClientHello: ) def __repr__(self): - return "ClientHello(sni: %s, alpn_protocols: %s, cipher_suites: %s)" % \ - (self.sni, self.alpn_protocols, self.cipher_suites) + return f"ClientHello(sni: {self.sni}, alpn_protocols: {self.alpn_protocols})" diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 096aae9f..3a60f79f 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -196,17 +196,14 @@ CIPHER_ID_NAME_MAP = { } # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. -# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +# https://ssl-config.mozilla.org/#config=old DEFAULT_CLIENT_CIPHERS = ( - "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" - "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" - "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" - "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" - "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" - "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" - "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" - "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" - "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + b"ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:" + b"ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:" + b"DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:" + b"ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:" + b"ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:" + b"AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" ) @@ -330,7 +327,7 @@ class TlsLayer(base.Layer): if self._custom_server_sni is False: return None else: - return self._custom_server_sni or self._client_hello and self._client_hello.sni + return self._custom_server_sni or self._client_hello and self._client_hello.sni.decode("idna") @property def alpn_for_client_connection(self): @@ -393,9 +390,9 @@ class TlsLayer(base.Layer): except exceptions.TlsException as e: raise exceptions.ClientHandshakeException( "Cannot establish TLS with client (sni: {sni}): {e}".format( - sni=self._client_hello.sni, e=repr(e) + sni=self._client_hello.sni.decode("idna"), e=repr(e) ), - self._client_hello.sni or repr(self.server_conn.address) + self._client_hello.sni.decode("idna") or repr(self.server_conn.address) ) def _establish_tls_with_server(self): @@ -493,7 +490,7 @@ class TlsLayer(base.Layer): organization = upstream_cert.organization # Also add SNI values. if self._client_hello.sni: - sans.add(self._client_hello.sni.encode("idna")) + sans.add(self._client_hello.sni) if self._custom_server_sni: sans.add(self._custom_server_sni.encode("idna")) diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index 4805f874..74a048ad 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -57,7 +57,7 @@ class RootContext: except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: - is_filtered = self.config.check_filter((client_hello.sni, 443)) + is_filtered = self.config.check_filter((client_hello.sni.decode("idna"), 443)) if is_filtered: return protocol.RawTCPLayer(top_layer, ignore=True) diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index 68e67dbe..c4e76bc6 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -116,7 +116,7 @@ class TestClientHello: ) c = tls.ClientHello(data) assert repr(c) - assert c.sni == 'example.com' + assert c.sni == b'example.com' assert c.cipher_suites == [ 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161, 49171, 49162, 49172, 156, 157, 47, 53, 10 diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 7c371c1e..c744e621 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -95,7 +95,7 @@ class TestServerConnection: def test_repr(self): c = tflow.tserver_conn() - c.sni = 'foobar' + c.sni = b'foobar' c.tls_established = True c.alpn_proto_negotiated = b'h2' assert 'address:22' in repr(c) -- cgit v1.2.3 From bdc15cbe0c0cd5175af1b58078d65d400cca71d1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 12 Nov 2019 02:59:01 +0100 Subject: update mypy --- examples/complex/xss_scanner.py | 4 +-- mitmproxy/addons/cut.py | 8 ++--- mitmproxy/addons/eventstore.py | 2 +- mitmproxy/addons/script.py | 2 +- mitmproxy/addons/session.py | 4 +-- mitmproxy/addons/stickycookie.py | 1 + mitmproxy/addons/view.py | 2 +- mitmproxy/certs.py | 2 +- mitmproxy/command.py | 4 ++- mitmproxy/contentviews/__init__.py | 4 ++- mitmproxy/contentviews/base.py | 4 +-- mitmproxy/contentviews/xml_html.py | 6 ++-- mitmproxy/ctx.py | 12 +++---- mitmproxy/flowfilter.py | 8 ++--- mitmproxy/http.py | 46 +++++++++++--------------- mitmproxy/io/tnetstring.py | 10 +++--- mitmproxy/net/http/message.py | 7 ++-- mitmproxy/net/tls.py | 3 +- mitmproxy/optmanager.py | 4 ++- mitmproxy/platform/__init__.py | 28 +++++++++------- mitmproxy/proxy/config.py | 6 ++-- mitmproxy/proxy/protocol/http2.py | 18 +++++----- mitmproxy/proxy/protocol/tls.py | 25 ++++++++------ mitmproxy/proxy/root_context.py | 3 +- mitmproxy/proxy/server.py | 2 +- mitmproxy/stateobject.py | 8 ++--- mitmproxy/tools/_main.py | 21 ++++++------ mitmproxy/tools/console/commander/commander.py | 2 +- mitmproxy/tools/console/common.py | 2 +- mitmproxy/tools/console/grideditor/base.py | 6 ++-- mitmproxy/tools/console/grideditor/editors.py | 6 ++-- mitmproxy/tools/console/palettes.py | 2 +- mitmproxy/tools/web/app.py | 7 ++-- mitmproxy/types.py | 2 +- mitmproxy/utils/sliding_window.py | 4 +-- mitmproxy/utils/strutils.py | 15 +++++---- mitmproxy/version.py | 4 +-- pathod/test.py | 2 +- setup.py | 4 +-- test/mitmproxy/test_connections.py | 2 +- 40 files changed, 154 insertions(+), 148 deletions(-) diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index 97e94ed4..d5f4aaab 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -86,7 +86,7 @@ def get_cookies(flow: http.HTTPFlow) -> Cookies: return {name: value for name, value in flow.request.cookies.fields} -def find_unclaimed_URLs(body: str, requestUrl: bytes) -> None: +def find_unclaimed_URLs(body, requestUrl): """ Look for unclaimed URLs in script tags and log them if found""" def getValue(attrs: List[Tuple[str, str]], attrName: str) -> Optional[str]: for name, value in attrs: @@ -111,7 +111,7 @@ def find_unclaimed_URLs(body: str, requestUrl: bytes) -> None: try: socket.gethostbyname(domain) except socket.gaierror: - ctx.log.error("XSS found in %s due to unclaimed URL \"%s\"." % (requestUrl, url)) + ctx.log.error(f"XSS found in {requestUrl} due to unclaimed URL \"{url}\".") def test_end_of_URL_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData: diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 6bb52e84..9aff2878 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -126,20 +126,18 @@ class Cut: format is UTF-8 encoded CSV. If there is exactly one row and one column, the data is written to file as-is, with raw bytes preserved. """ + v: typing.Union[str, bytes] fp = io.StringIO(newline="") if len(cuts) == 1 and len(flows) == 1: v = extract(cuts[0], flows[0]) - if isinstance(v, bytes): - fp.write(strutils.always_str(v)) - else: - fp.write(v) + fp.write(strutils.always_str(v)) # type: ignore ctx.log.alert("Clipped single cut.") else: writer = csv.writer(fp) for f in flows: vals = [extract(c, f) for c in cuts] writer.writerow( - [strutils.always_str(v) or "" for v in vals] # type: ignore + [strutils.always_str(v) for v in vals] ) ctx.log.alert("Clipped %s cuts as CSV." % len(cuts)) try: diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index 50fea7ab..188a3b39 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -14,7 +14,7 @@ class EventStore: self.sig_refresh = blinker.Signal() @property - def size(self) -> int: + def size(self) -> typing.Optional[int]: return self.data.maxlen def log(self, entry: LogEntry) -> None: diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index a39ce5ce..3b2568c9 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -16,7 +16,7 @@ from mitmproxy import ctx import mitmproxy.types as mtypes -def load_script(path: str) -> types.ModuleType: +def load_script(path: str) -> typing.Optional[types.ModuleType]: fullname = "__mitmproxy_script__.{}".format( os.path.splitext(os.path.basename(path))[0] ) diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py index f9073c3e..6636b500 100644 --- a/mitmproxy/addons/session.py +++ b/mitmproxy/addons/session.py @@ -215,8 +215,8 @@ class Session: def __init__(self): self.db_store: SessionDB = None self._hot_store: collections.OrderedDict = collections.OrderedDict() - self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str]]] = {} - self._view: typing.List[typing.Tuple[typing.Union[int, float, str], str]] = [] + self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str, None]]] = {} + self._view: typing.List[typing.Tuple[typing.Union[int, float, str, None], str]] = [] self.order: str = orders[0] self.filter = matchall self._flush_period: float = self._FP_DEFAULT diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index fd530aaa..1651c1f6 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -53,6 +53,7 @@ class StickyCookie: self.flt = None def response(self, flow: http.HTTPFlow): + assert flow.response if self.flt: for name, (value, attrs) in flow.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 8d27840f..4224877a 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -584,7 +584,7 @@ class Focus: """ def __init__(self, v: View) -> None: self.view = v - self._flow: mitmproxy.flow.Flow = None + self._flow: typing.Optional[mitmproxy.flow.Flow] = None self.sig_change = blinker.Signal() if len(self.view): self.flow = self.view[0] diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index e702e1cf..d574c027 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -123,7 +123,7 @@ def dummy_cert(privkey, cacert, commonname, sans, organization): ) ]) cert.set_pubkey(cacert.get_pubkey()) - cert.sign(privkey, b"sha256") + cert.sign(privkey, "sha256") return Cert(cert) diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 27f0921d..0998601c 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -44,6 +44,8 @@ def typename(t: type) -> str: class Command: + returntype: typing.Optional[typing.Type] + def __init__(self, manager, path, func) -> None: self.path = path self.manager = manager @@ -177,7 +179,7 @@ class CommandManager(mitmproxy.types._CommandBase): parse: typing.List[ParseResult] = [] params: typing.List[type] = [] - typ: typing.Type = None + typ: typing.Type for i in range(len(parts)): if i == 0: typ = mitmproxy.types.Cmd diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 01c6d221..1e71d942 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -135,7 +135,9 @@ def get_content_view(viewmode: View, data: bytes, **metadata): # Third-party viewers can fail in unexpected ways... except Exception: desc = "Couldn't parse: falling back to Raw" - _, content = get("Raw")(data, **metadata) + raw = get("Raw") + assert raw + content = raw(data, **metadata)[1] error = "{} Content viewer failed: \n{}".format( getattr(viewmode, "name"), traceback.format_exc() diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 9b34f3d4..81f2e487 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -9,8 +9,8 @@ TViewResult = typing.Tuple[str, typing.Iterator[TViewLine]] class View: - name: str = None - content_types: typing.List[str] = [] + name: typing.ClassVar[str] + content_types: typing.ClassVar[typing.List[str]] = [] def __call__(self, data: bytes, **metadata) -> TViewResult: """ diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 00a62a15..f2fa47cb 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -1,7 +1,7 @@ import io import re import textwrap -from typing import Iterable +from typing import Iterable, Optional from mitmproxy.contentviews import base from mitmproxy.utils import sliding_window @@ -124,14 +124,14 @@ def indent_text(data: str, prefix: str) -> str: return textwrap.indent(dedented, prefix[:32]) -def is_inline_text(a: Token, b: Token, c: Token) -> bool: +def is_inline_text(a: Optional[Token], b: Optional[Token], c: Optional[Token]) -> bool: if isinstance(a, Tag) and isinstance(b, Text) and isinstance(c, Tag): if a.is_opening and "\n" not in b.data and c.is_closing and a.tag == c.tag: return True return False -def is_inline(prev2: Token, prev1: Token, t: Token, next1: Token, next2: Token) -> bool: +def is_inline(prev2: Optional[Token], prev1: Optional[Token], t: Optional[Token], next1: Optional[Token], next2: Optional[Token]) -> bool: if isinstance(t, Text): return is_inline_text(prev1, t, next1) elif isinstance(t, Tag): diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py index 5df6f9c1..2ce9c7c2 100644 --- a/mitmproxy/ctx.py +++ b/mitmproxy/ctx.py @@ -1,7 +1,7 @@ -import mitmproxy.master # noqa -import mitmproxy.log # noqa -import mitmproxy.options # noqa +import mitmproxy.log +import mitmproxy.master +import mitmproxy.options -master = None # type: mitmproxy.master.Master -log: mitmproxy.log.Log = None -options: mitmproxy.options.Options = None +log: "mitmproxy.log.Log" +master: "mitmproxy.master.Master" +options: "mitmproxy.options.Options" diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 0d8f1062..3f5afb48 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -44,7 +44,7 @@ from mitmproxy import flow from mitmproxy.utils import strutils import pyparsing as pp -from typing import Callable, Sequence, Type # noqa +from typing import Callable, Sequence, Type, Optional, ClassVar def only(*types): @@ -69,8 +69,8 @@ class _Token: class _Action(_Token): - code: str = None - help: str = None + code: ClassVar[str] + help: ClassVar[str] @classmethod def make(klass, s, loc, toks): @@ -539,7 +539,7 @@ bnf = _make() TFilter = Callable[[flow.Flow], bool] -def parse(s: str) -> TFilter: +def parse(s: str) -> Optional[TFilter]: try: flt = bnf.parseString(s, parseAll=True)[0] flt.pattern = s diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 3c16b807..6b527e75 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1,15 +1,13 @@ import html from typing import Optional +from mitmproxy import connections from mitmproxy import flow - -from mitmproxy.net import http from mitmproxy import version -from mitmproxy import connections # noqa +from mitmproxy.net import http class HTTPRequest(http.Request): - """ A mitmproxy HTTP request. """ @@ -85,10 +83,10 @@ class HTTPRequest(http.Request): class HTTPResponse(http.Response): - """ A mitmproxy HTTP response. """ + # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and # may be removed in the future. @@ -136,34 +134,28 @@ class HTTPResponse(http.Response): class HTTPFlow(flow.Flow): - """ An HTTPFlow is a collection of objects representing a single HTTP transaction. """ + request: HTTPRequest + response: Optional[HTTPResponse] = None + error: Optional[flow.Error] = None + """ + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + """ + server_conn: connections.ServerConnection + client_conn: connections.ClientConnection + intercepted: bool = False + """ Is this flow currently being intercepted? """ + mode: str + """ What mode was the proxy layer in when receiving this request? """ def __init__(self, client_conn, server_conn, live=None, mode="regular"): super().__init__("http", client_conn, server_conn, live) - - self.request: HTTPRequest = None - """ :py:class:`HTTPRequest` object """ - self.response: HTTPResponse = None - """ :py:class:`HTTPResponse` object """ - self.error: flow.Error = None - """ :py:class:`Error` object - - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. - """ - self.server_conn: connections.ServerConnection = server_conn - """ :py:class:`ServerConnection` object """ - self.client_conn: connections.ClientConnection = client_conn - """:py:class:`ClientConnection` object """ - self.intercepted: bool = False - """ Is this flow currently being intercepted? """ self.mode = mode - """ What mode was the proxy layer in when receiving this request? """ _stateobject_attributes = flow.Flow._stateobject_attributes.copy() # mypy doesn't support update with kwargs @@ -205,8 +197,8 @@ class HTTPFlow(flow.Flow): def make_error_response( status_code: int, - message: str="", - headers: Optional[http.Headers]=None, + message: str = "", + headers: Optional[http.Headers] = None, ) -> HTTPResponse: reason = http.status_codes.RESPONSES.get(status_code, "Unknown") body = """ diff --git a/mitmproxy/io/tnetstring.py b/mitmproxy/io/tnetstring.py index aa1f5670..de84279b 100644 --- a/mitmproxy/io/tnetstring.py +++ b/mitmproxy/io/tnetstring.py @@ -192,22 +192,22 @@ def parse(data_type: int, data: bytes) -> TSerializable: try: return int(data) except ValueError: - raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) + raise ValueError(f"not a tnetstring: invalid integer literal: {data!r}") if data_type == ord(b'^'): try: return float(data) except ValueError: - raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) + raise ValueError(f"not a tnetstring: invalid float literal: {data!r}") if data_type == ord(b'!'): if data == b'true': return True elif data == b'false': return False else: - raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) + raise ValueError(f"not a tnetstring: invalid boolean literal: {data!r}") if data_type == ord(b'~'): if data: - raise ValueError("not a tnetstring: invalid null literal") + raise ValueError(f"not a tnetstring: invalid null literal: {data!r}") return None if data_type == ord(b']'): l = [] @@ -236,7 +236,7 @@ def pop(data: bytes) -> typing.Tuple[TSerializable, bytes]: blength, data = data.split(b':', 1) length = int(blength) except ValueError: - raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(data)) + raise ValueError(f"not a tnetstring: missing or invalid length prefix: {data!r}") try: data, data_type, remain = data[:length], data[length], data[length + 1:] except IndexError: diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index d5a7ff9c..af7b032b 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -82,7 +82,7 @@ class Message(serializable.Serializable): def raw_content(self, content): self.data.content = content - def get_content(self, strict: bool=True) -> bytes: + def get_content(self, strict: bool=True) -> Optional[bytes]: """ The uncompressed HTTP message body as bytes. @@ -195,10 +195,9 @@ class Message(serializable.Serializable): See also: :py:attr:`content`, :py:class:`raw_content` """ - if self.raw_content is None: - return None - content = self.get_content(strict) + if content is None: + return None enc = self._guess_encoding(content) try: return encoding.decode(content, enc) diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 48392d1b..d68a008f 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -474,8 +474,7 @@ class ClientHello: return cls(raw_client_hello) except EOFError as e: raise exceptions.TlsProtocolException( - 'Cannot parse Client Hello: %s, Raw Client Hello: %s' % - (repr(e), binascii.hexlify(raw_client_hello)) + f"Cannot parse Client Hello: {e!r}, Raw Client Hello: {binascii.hexlify(raw_client_hello)!r}" ) def __repr__(self): diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 6e187b0d..f42aa645 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -551,7 +551,9 @@ def serialize(opts: OptManager, text: str, defaults: bool = False) -> str: for k in list(data.keys()): if k not in opts._options: del data[k] - return ruamel.yaml.round_trip_dump(data) + ret = ruamel.yaml.round_trip_dump(data) + assert ret + return ret def save(opts: OptManager, path: str, defaults: bool =False) -> None: diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py index 61946ec4..7e690789 100644 --- a/mitmproxy/platform/__init__.py +++ b/mitmproxy/platform/__init__.py @@ -1,7 +1,7 @@ import re import socket import sys -from typing import Tuple +from typing import Callable, Optional, Tuple def init_transparent_mode() -> None: @@ -10,30 +10,34 @@ def init_transparent_mode() -> None: """ -def original_addr(csock: socket.socket) -> Tuple[str, int]: - """ - Get the original destination for the given socket. - This function will be None if transparent mode is not supported. - """ - +original_addr: Optional[Callable[[socket.socket], Tuple[str, int]]] +""" +Get the original destination for the given socket. +This function will be None if transparent mode is not supported. +""" if re.match(r"linux(?:2)?", sys.platform): from . import linux - original_addr = linux.original_addr # noqa + original_addr = linux.original_addr elif sys.platform == "darwin" or sys.platform.startswith("freebsd"): from . import osx - original_addr = osx.original_addr # noqa + original_addr = osx.original_addr elif sys.platform.startswith("openbsd"): from . import openbsd - original_addr = openbsd.original_addr # noqa + original_addr = openbsd.original_addr elif sys.platform == "win32": from . import windows resolver = windows.Resolver() init_transparent_mode = resolver.setup # noqa - original_addr = resolver.original_addr # noqa + original_addr = resolver.original_addr else: - original_addr = None # noqa + original_addr = None + +__all__ = [ + "original_addr", + "init_transparent_mode" +] diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index ae2ec68b..e98faabf 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -34,9 +34,9 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_filter: HostMatcher = None - self.check_tcp: HostMatcher = None - self.certstore: certs.CertStore = None + self.certstore: certs.CertStore + self.check_filter: typing.Optional[HostMatcher] = None + self.check_tcp: typing.Optional[HostMatcher] = None self.upstream_server: typing.Optional[server_spec.ServerSpec] = None self.configure(options, set(options.keys())) options.changed.connect(self.configure) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 42b61f4d..a5870e6c 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -1,7 +1,7 @@ import threading import time import functools -from typing import Dict, Callable, Any, List # noqa +from typing import Dict, Callable, Any, List, Optional # noqa import h2.exceptions from h2 import connection @@ -382,15 +382,15 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr ctx, name="Http2SingleStreamLayer-{}".format(stream_id) ) self.h2_connection = h2_connection - self.zombie: float = None + self.zombie: Optional[float] = None self.client_stream_id: int = stream_id - self.server_stream_id: int = None + self.server_stream_id: Optional[int] = None self.request_headers = request_headers - self.response_headers: mitmproxy.net.http.Headers = None + self.response_headers: Optional[mitmproxy.net.http.Headers] = None self.pushed = False - self.timestamp_start: float = None - self.timestamp_end: float = None + self.timestamp_start: Optional[float] = None + self.timestamp_end: Optional[float] = None self.request_arrived = threading.Event() self.request_data_queue: queue.Queue[bytes] = queue.Queue() @@ -404,9 +404,9 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.no_body = False - self.priority_exclusive: bool = None - self.priority_depends_on: int = None - self.priority_weight: int = None + self.priority_exclusive: bool + self.priority_depends_on: Optional[int] = None + self.priority_weight: Optional[int] = None self.handled_priority_event: Any = None def kill(self): diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 3a60f79f..282df60d 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -198,12 +198,12 @@ CIPHER_ID_NAME_MAP = { # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://ssl-config.mozilla.org/#config=old DEFAULT_CLIENT_CIPHERS = ( - b"ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:" - b"ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:" - b"DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:" - b"ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:" - b"ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:" - b"AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:" + "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:" + "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:" + "ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:" + "ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:" + "AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" ) @@ -320,14 +320,18 @@ class TlsLayer(base.Layer): return self._server_tls @property - def server_sni(self): + def server_sni(self) -> Optional[str]: """ The Server Name Indication we want to send with the next server TLS handshake. """ if self._custom_server_sni is False: return None + elif self._custom_server_sni: + return self._custom_server_sni + elif self._client_hello and self._client_hello.sni: + return self._client_hello.sni.decode("idna") else: - return self._custom_server_sni or self._client_hello and self._client_hello.sni.decode("idna") + return None @property def alpn_for_client_connection(self): @@ -388,11 +392,12 @@ class TlsLayer(base.Layer): # raises ann error. self.client_conn.rfile.peek(1) except exceptions.TlsException as e: + sni_str = self._client_hello.sni and self._client_hello.sni.decode("idna") raise exceptions.ClientHandshakeException( "Cannot establish TLS with client (sni: {sni}): {e}".format( - sni=self._client_hello.sni.decode("idna"), e=repr(e) + sni=sni_str, e=repr(e) ), - self._client_hello.sni.decode("idna") or repr(self.server_conn.address) + sni_str or repr(self.server_conn.address) ) def _establish_tls_with_server(self): diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index 74a048ad..3d4e8660 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -57,7 +57,8 @@ class RootContext: except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: - is_filtered = self.config.check_filter((client_hello.sni.decode("idna"), 443)) + sni_str = client_hello.sni and client_hello.sni.decode("idna") + is_filtered = self.config.check_filter((sni_str, 443)) if is_filtered: return protocol.RawTCPLayer(top_layer, ignore=True) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 44ae5697..3688b677 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -35,6 +35,7 @@ class DummyServer: class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True + channel: controller.Channel def __init__(self, config: config.ProxyConfig) -> None: """ @@ -53,7 +54,6 @@ class ProxyServer(tcp.TCPServer): raise exceptions.ServerException( 'Error starting proxy server: ' + repr(e) ) from e - self.channel: controller.Channel = None def set_channel(self, channel): self.channel = channel diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 2c16dcda..76329236 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,7 +1,5 @@ -import typing -from typing import Any # noqa -from typing import MutableMapping # noqa import json +import typing from mitmproxy.coretypes import serializable from mitmproxy.utils import typecheck @@ -15,7 +13,7 @@ class StateObject(serializable.Serializable): or StateObject instances themselves. """ - _stateobject_attributes: MutableMapping[str, Any] = None + _stateobject_attributes: typing.ClassVar[typing.MutableMapping[str, typing.Any]] """ An attribute-name -> class-or-type dict containing all attributes that should be serialized. If the attribute is a class, it must implement the @@ -42,7 +40,7 @@ class StateObject(serializable.Serializable): if val is None: setattr(self, attr, val) else: - curr = getattr(self, attr) + curr = getattr(self, attr, None) if hasattr(curr, "set_state"): curr.set_state(val) else: diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index b95d73ab..8a98278d 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -6,17 +6,16 @@ Feel free to import and use whatever new package you deem necessary. import os import sys import asyncio -import argparse # noqa -import signal # noqa -import typing # noqa - -from mitmproxy.tools import cmdline # noqa -from mitmproxy import exceptions, master # noqa -from mitmproxy import options # noqa -from mitmproxy import optmanager # noqa -from mitmproxy import proxy # noqa -from mitmproxy import log # noqa -from mitmproxy.utils import debug, arg_check # noqa +import argparse +import signal +import typing + +from mitmproxy.tools import cmdline +from mitmproxy import exceptions, master +from mitmproxy import options +from mitmproxy import optmanager +from mitmproxy import proxy +from mitmproxy.utils import debug, arg_check OPTIONS_FILE_NAME = "config.yaml" diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index e8550f86..f291b8fd 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -55,7 +55,7 @@ class CommandBuffer: self.text = self.flatten(start) # Cursor is always within the range [0:len(buffer)]. self._cursor = len(self.text) - self.completion: CompletionState = None + self.completion: typing.Optional[CompletionState] = None @property def cursor(self) -> int: diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 43ab50cb..3a5b4aeb 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -38,7 +38,7 @@ KEY_MAX = 30 def format_keyvals( - entries: typing.List[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]], + entries: typing.Iterable[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]], key_format: str = "key", value_format: str = "text", indent: int = 0 diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py index 3badf1a6..64b6e5d5 100644 --- a/mitmproxy/tools/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -254,7 +254,7 @@ FIRST_WIDTH_MAX = 40 class BaseGridEditor(urwid.WidgetWrap): - title = "" + title: str = "" keyctx = "grideditor" def __init__( @@ -402,8 +402,8 @@ class BaseGridEditor(urwid.WidgetWrap): class GridEditor(BaseGridEditor): - title: str = None - columns: typing.Sequence[Column] = None + title = "" + columns: typing.Sequence[Column] = () keyctx = "grideditor" def __init__( diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 61fcf6b4..b4d59384 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -107,7 +107,7 @@ class CookieAttributeEditor(base.FocusEditor): col_text.Column("Name"), col_text.Column("Value"), ] - grideditor: base.BaseGridEditor = None + grideditor: base.BaseGridEditor def data_in(self, data): return [(k, v or "") for k, v in data] @@ -169,7 +169,7 @@ class SetCookieEditor(base.FocusEditor): class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget): - title: str = None + title = "" columns = [ col_text.Column("") ] @@ -189,7 +189,7 @@ class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget): class DataViewer(base.GridEditor, layoutwidget.LayoutWidget): - title: str = None + title = "" def __init__( self, diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 4eee7692..6033ff25 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -42,7 +42,7 @@ class Palette: 'commander_command', 'commander_invalid', 'commander_hint' ] _fields.extend(['gradient_%02d' % i for i in range(100)]) - high: typing.Mapping[str, typing.Sequence[str]] = None + high: typing.Optional[typing.Mapping[str, typing.Sequence[str]]] = None def palette(self, transparent): l = [] diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index acf2cfdf..a0803755 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -5,6 +5,7 @@ import logging import os.path import re from io import BytesIO +from typing import ClassVar, Optional import tornado.escape import tornado.web @@ -50,6 +51,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: f["error"] = flow.error.get_state() if isinstance(flow, http.HTTPFlow): + content_length: Optional[int] + content_hash: Optional[str] if flow.request: if flow.request.raw_content: content_length = len(flow.request.raw_content) @@ -193,7 +196,7 @@ class FilterHelp(RequestHandler): class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): # raise an error if inherited class doesn't specify its own instance. - connections: set = None + connections: ClassVar[set] def open(self): self.connections.add(self) @@ -213,7 +216,7 @@ class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): class ClientConnection(WebSocketEventBroadcaster): - connections: set = set() + connections: ClassVar[set] = set() class Flows(RequestHandler): diff --git a/mitmproxy/types.py b/mitmproxy/types.py index f2a26b40..0634e4d7 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -423,7 +423,7 @@ class TypeManager: for t in types: self.typemap[t.typ] = t() - def get(self, t: type, default=None) -> _BaseType: + def get(self, t: typing.Optional[typing.Type], default=None) -> _BaseType: if type(t) in self.typemap: return self.typemap[type(t)] return self.typemap.get(t, default) diff --git a/mitmproxy/utils/sliding_window.py b/mitmproxy/utils/sliding_window.py index 0a65f5e4..cb31756d 100644 --- a/mitmproxy/utils/sliding_window.py +++ b/mitmproxy/utils/sliding_window.py @@ -1,5 +1,5 @@ import itertools -from typing import TypeVar, Iterable, Iterator, Tuple, Optional +from typing import TypeVar, Iterable, Iterator, Tuple, Optional, List T = TypeVar('T') @@ -18,7 +18,7 @@ def window(iterator: Iterable[T], behind: int = 0, ahead: int = 0) -> Iterator[T 2 3 None """ # TODO: move into utils - iters = list(itertools.tee(iterator, behind + 1 + ahead)) + iters: List[Iterator[Optional[T]]] = list(itertools.tee(iterator, behind + 1 + ahead)) for i in range(behind): iters[i] = itertools.chain((behind - i) * [None], iters[i]) for i in range(ahead): diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 388c765f..6e399d8f 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,10 +1,10 @@ +import codecs import io import re -import codecs -from typing import AnyStr, Optional, cast, Iterable +from typing import Iterable, Optional, Union, cast -def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: +def always_bytes(str_or_bytes: Union[str, bytes, None], *encode_args) -> Optional[bytes]: if isinstance(str_or_bytes, bytes) or str_or_bytes is None: return cast(Optional[bytes], str_or_bytes) elif isinstance(str_or_bytes, str): @@ -13,13 +13,15 @@ def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes raise TypeError("Expected str or bytes, but got {}.".format(type(str_or_bytes).__name__)) -def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]: +def always_str(str_or_bytes: Union[str, bytes, None], *decode_args) -> Optional[str]: """ Returns, str_or_bytes unmodified, if """ - if isinstance(str_or_bytes, str) or str_or_bytes is None: - return cast(Optional[str], str_or_bytes) + if str_or_bytes is None: + return None + if isinstance(str_or_bytes, str): + return cast(str, str_or_bytes) elif isinstance(str_or_bytes, bytes): return str_or_bytes.decode(*decode_args) else: @@ -39,7 +41,6 @@ _control_char_trans_newline = _control_char_trans.copy() for x in ("\r", "\n", "\t"): del _control_char_trans_newline[ord(x)] - _control_char_trans = str.maketrans(_control_char_trans) _control_char_trans_newline = str.maketrans(_control_char_trans_newline) diff --git a/mitmproxy/version.py b/mitmproxy/version.py index b40fae8b..363a4bf6 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -25,9 +25,9 @@ def get_dev_version() -> str: stderr=subprocess.STDOUT, cwd=here, ) - last_tag, tag_dist, commit = git_describe.decode().strip().rsplit("-", 2) + last_tag, tag_dist_str, commit = git_describe.decode().strip().rsplit("-", 2) commit = commit.lstrip("g")[:7] - tag_dist = int(tag_dist) + tag_dist = int(tag_dist_str) except Exception: pass else: diff --git a/pathod/test.py b/pathod/test.py index e8c3c84a..b6e5e4d0 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -25,7 +25,7 @@ class Daemon: def __enter__(self): return self - def __exit__(self, type, value, traceback) -> bool: + def __exit__(self, type, value, traceback): self.logfp.truncate(0) self.shutdown() return False diff --git a/setup.py b/setup.py index 7c5f1eb2..9343dd99 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setup( "kaitaistruct>=0.7,<0.9", "ldap3>=2.6.1,<2.7", "passlib>=1.6.5, <1.8", - "protobuf>=3.6.0, <3.10", + "protobuf>=3.6.0, <3.11", "pyasn1>=0.3.1,<0.5", "pyOpenSSL>=19.0.0,<20", "pyparsing>=2.4.2,<2.5", @@ -93,7 +93,7 @@ setup( "asynctest>=0.12.0", "flake8>=3.7.8,<3.8", "Flask>=1.0,<1.2", - "mypy>=0.590,<0.591", + "mypy>=0.740,<0.741", "parver>=0.1,<2.0", "pytest-asyncio>=0.10.0,<0.11", "pytest-cov>=2.7.1,<3", diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index c744e621..7c371c1e 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -95,7 +95,7 @@ class TestServerConnection: def test_repr(self): c = tflow.tserver_conn() - c.sni = b'foobar' + c.sni = 'foobar' c.tls_established = True c.alpn_proto_negotiated = b'h2' assert 'address:22' in repr(c) -- cgit v1.2.3 From 836e04abd6b39390bbe9112095b81b4eee74c610 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Thu, 14 Nov 2019 18:39:06 +0100 Subject: adding config.yml as an allowed config filename cf. title as suggested in #3639 --- mitmproxy/tools/_main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index 8a98278d..bcec7327 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -17,8 +17,6 @@ from mitmproxy import optmanager from mitmproxy import proxy from mitmproxy.utils import debug, arg_check -OPTIONS_FILE_NAME = "config.yaml" - def assert_utf8_env(): spec = "" @@ -69,6 +67,8 @@ def run( options. """ debug.register_info_dumpers() + # Be tolerant as the yml is often recommended + OPTIONS_FILE_NAMES = ["config.yaml", "config.yml"] opts = options.Options() master = master_cls(opts) @@ -87,10 +87,11 @@ def run( sys.exit(1) try: opts.set(*args.setoptions, defer=True) - optmanager.load_paths( - opts, - os.path.join(opts.confdir, OPTIONS_FILE_NAME), - ) + for conf_file in OPTIONS_FILE_NAMES: + optmanager.load_paths( + opts, + os.path.join(opts.confdir, conf_file), + ) pconf = process_options(parser, opts, args) server: typing.Any = None if pconf.options.server: -- cgit v1.2.3 From 9a1ec6b0646e67a8d9332910efec35752d003561 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Thu, 14 Nov 2019 18:59:43 +0100 Subject: Fixes #3694 --- mitmproxy/addons/view.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 4224877a..da9d19f9 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -238,18 +238,24 @@ class View(collections.abc.Sequence): """ Set focus to the next flow. """ - idx = self.focus.index + 1 - if self.inbounds(idx): - self.focus.flow = self[idx] + if self.focus.index is not None: + idx = self.focus.index + 1 + if self.inbounds(idx): + self.focus.flow = self[idx] + else: + pass @command.command("view.focus.prev") def focus_prev(self) -> None: """ Set focus to the previous flow. """ - idx = self.focus.index - 1 - if self.inbounds(idx): - self.focus.flow = self[idx] + if self.focus.index is not None: + idx = self.focus.index - 1 + if self.inbounds(idx): + self.focus.flow = self[idx] + else: + pass # Order @command.command("view.order.options") -- cgit v1.2.3 From c78165bd4c4e3ad89ed603d702c51f6b043b3b60 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 14 Nov 2019 21:12:32 +0100 Subject: simplify option file loading - no need for a constant that is only used once - if load_paths allows us to specify multiple paths, let's just do that. --- mitmproxy/tools/_main.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index bcec7327..a00a3e98 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -67,8 +67,6 @@ def run( options. """ debug.register_info_dumpers() - # Be tolerant as the yml is often recommended - OPTIONS_FILE_NAMES = ["config.yaml", "config.yml"] opts = options.Options() master = master_cls(opts) @@ -87,11 +85,11 @@ def run( sys.exit(1) try: opts.set(*args.setoptions, defer=True) - for conf_file in OPTIONS_FILE_NAMES: - optmanager.load_paths( - opts, - os.path.join(opts.confdir, conf_file), - ) + optmanager.load_paths( + opts, + os.path.join(opts.confdir, "config.yaml"), + os.path.join(opts.confdir, "config.yml"), + ) pconf = process_options(parser, opts, args) server: typing.Any = None if pconf.options.server: -- cgit v1.2.3 From 2239c49e18c62331a21aa75fa5e74e6a7ea88334 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 02:27:33 +0100 Subject: improve flowfilter --- mitmproxy/flowfilter.py | 112 +++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 64 deletions(-) diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 3f5afb48..b222d2a8 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -32,19 +32,17 @@ rex Equivalent to ~u rex """ +import functools import re import sys -import functools +from typing import Callable, ClassVar, Optional, Sequence, Type + +import pyparsing as pp +from mitmproxy import flow from mitmproxy import http -from mitmproxy import websocket from mitmproxy import tcp -from mitmproxy import flow - -from mitmproxy.utils import strutils - -import pyparsing as pp -from typing import Callable, Sequence, Type, Optional, ClassVar +from mitmproxy import websocket def only(*types): @@ -54,7 +52,9 @@ def only(*types): if isinstance(flow, types): return fn(self, flow) return False + return filter_types + return decorator @@ -146,10 +146,10 @@ class _Rex(_Action): def __init__(self, expr): self.expr = expr if self.is_binary: - expr = strutils.escaped_str_to_bytes(expr) + expr = expr.encode() try: self.re = re.compile(expr, self.flags) - except: + except Exception: raise ValueError("Cannot compile expression.") @@ -336,6 +336,7 @@ class FUrl(_Rex): code = "u" help = "URL" is_binary = False + # FUrl is special, because it can be "naked". @classmethod @@ -469,68 +470,51 @@ def _make(): # Order is important - multi-char expressions need to come before narrow # ones. parts = [] - for klass in filter_unary: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() - f.setParseAction(klass.make) + for cls in filter_unary: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + f.setParseAction(cls.make) parts.append(f) - simplerex = "".join(c for c in pp.printables if c not in "()~'\"") - alphdevanagari = pp.pyparsing_unicode.Devanagari.alphas - alphcyrillic = pp.pyparsing_unicode.Cyrillic.alphas - alphgreek = pp.pyparsing_unicode.Greek.alphas - alphchinese = pp.pyparsing_unicode.Chinese.alphas - alpharabic = pp.pyparsing_unicode.Arabic.alphas - alphhebrew = pp.pyparsing_unicode.Hebrew.alphas - alphjapanese = pp.pyparsing_unicode.Japanese.alphas - alphkorean = pp.pyparsing_unicode.Korean.alphas - alphlatin1 = pp.pyparsing_unicode.Latin1.alphas - alphlatinA = pp.pyparsing_unicode.LatinA.alphas - alphlatinB = pp.pyparsing_unicode.LatinB.alphas - - rex = pp.Word(simplerex) |\ - pp.Word(alphcyrillic) |\ - pp.Word(alphgreek) |\ - pp.Word(alphchinese) |\ - pp.Word(alpharabic) |\ - pp.Word(alphdevanagari) |\ - pp.Word(alphhebrew) |\ - pp.Word(alphjapanese) |\ - pp.Word(alphkorean) |\ - pp.Word(alphlatin1) |\ - pp.Word(alphlatinA) |\ - pp.Word(alphlatinB) |\ - pp.QuotedString("\"", escChar='\\') |\ - pp.QuotedString("'", escChar='\\') - for klass in filter_rex: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy() - f.setParseAction(klass.make) + # This is a bit of a hack to simulate Word(pyparsing_unicode.printables), + # which has a horrible performance with len(pyparsing.pyparsing_unicode.printables) == 1114060 + unicode_words = pp.CharsNotIn("()~'\"" + pp.ParserElement.DEFAULT_WHITE_CHARS) + unicode_words.skipWhitespace = True + regex = ( + unicode_words + | pp.QuotedString('"', escChar='\\') + | pp.QuotedString("'", escChar='\\') + ) + for cls in filter_rex: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + regex.copy() + f.setParseAction(cls.make) parts.append(f) - for klass in filter_int: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums) - f.setParseAction(klass.make) + for cls in filter_int: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + pp.Word(pp.nums) + f.setParseAction(cls.make) parts.append(f) # A naked rex is a URL rex: - f = rex.copy() + f = regex.copy() f.setParseAction(FUrl.make) parts.append(f) atom = pp.MatchFirst(parts) - expr = pp.operatorPrecedence(atom, - [(pp.Literal("!").suppress(), - 1, - pp.opAssoc.RIGHT, - lambda x: FNot(*x)), - (pp.Literal("&").suppress(), - 2, - pp.opAssoc.LEFT, - lambda x: FAnd(*x)), - (pp.Literal("|").suppress(), - 2, - pp.opAssoc.LEFT, - lambda x: FOr(*x)), - ]) + expr = pp.infixNotation( + atom, + [(pp.Literal("!").suppress(), + 1, + pp.opAssoc.RIGHT, + lambda x: FNot(*x)), + (pp.Literal("&").suppress(), + 2, + pp.opAssoc.LEFT, + lambda x: FAnd(*x)), + (pp.Literal("|").suppress(), + 2, + pp.opAssoc.LEFT, + lambda x: FOr(*x)), + ]) expr = pp.OneOrMore(expr) return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x) @@ -570,15 +554,15 @@ def match(flt, flow): help = [] for a in filter_unary: help.append( - ("~%s" % a.code, a.help) + (f"~{a.code}", a.help) ) for b in filter_rex: help.append( - ("~%s regex" % b.code, b.help) + (f"~{b.code} regex", b.help) ) for c in filter_int: help.append( - ("~%s int" % c.code, c.help) + (f"~{c.code} int", c.help) ) help.sort() help.extend( -- cgit v1.2.3 From be46008b5ef0d71b65b8f4dab7528b4bcc7205e6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 15:56:55 +0100 Subject: disable overly strict indentation checks --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0a34b805..e7643b08 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [flake8] max-line-length = 140 max-complexity = 25 -ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741 +ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741,E126 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/* addons = file,open,basestring,xrange,unicode,long,cmp -- cgit v1.2.3 From a5412ab1366205cb1a2000993cb6b2c9dca28246 Mon Sep 17 00:00:00 2001 From: Jurriaan Bremer Date: Sun, 29 Apr 2018 16:53:11 +0200 Subject: allow server replay functionality to run on a different port By providing the "server_replay_ignore_port" configuration value we're able to run mitmproxy in server replay mode on a different port than the web server in the flows was originally running. This is also useful in case multiple ports are present in a flow (I suspect). --- mitmproxy/addons/serverplayback.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 51ba60b4..0818696f 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -68,6 +68,13 @@ class ServerPlayback: to replay. """ ) + loader.add_option( + "server_replay_ignore_port", bool, False, + """ + Ignore request's destination port while searching for a saved flow + to replay. + """ + ) @command.command("replay.server") def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: @@ -110,7 +117,7 @@ class ServerPlayback: _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) - key: typing.List[typing.Any] = [str(r.port), str(r.scheme), str(r.method), str(path)] + key: typing.List[typing.Any] = [str(r.scheme), str(r.method), str(path)] if not ctx.options.server_replay_ignore_content: if ctx.options.server_replay_ignore_payload_params and r.multipart_form: key.extend( @@ -129,6 +136,8 @@ class ServerPlayback: if not ctx.options.server_replay_ignore_host: key.append(r.host) + if not ctx.options.server_replay_ignore_port: + key.append(r.port) filtered = [] ignore_params = ctx.options.server_replay_ignore_params or [] -- cgit v1.2.3 From f8926170a5be481ff4cb2dd181ead95b0ec081cb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 17:22:41 +0100 Subject: remove superfluous option --- mitmproxy/tools/console/consoleaddons.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index ba14eeb6..a40cdeaa 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -113,11 +113,6 @@ class ConsoleAddon: "console_mouse", bool, True, "Console mouse interaction." ) - loader.add_option( - "console_external_viewer_default", str, "vi", - "External viewer for flow body. Set environment variable $MITMPROXY_EDITOR " - "to change default." - ) @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: -- cgit v1.2.3