diff options
author | Maximilian Hils <git@maximilianhils.com> | 2019-11-15 18:10:42 +0100 |
---|---|---|
committer | Maximilian Hils <git@maximilianhils.com> | 2019-11-15 18:10:42 +0100 |
commit | 0873566ff05c02be063f3aa15adecb725342119c (patch) | |
tree | a6dcddf0f0c08c9ff9cdd89e1f95196af2ff5a6a /mitmproxy/addons | |
parent | d852f292c9a45de7f45cc8537f2aef217259017e (diff) | |
parent | fbbbbb678e77e3cff5ab35b1fd55e85e875ed3f5 (diff) | |
download | mitmproxy-0873566ff05c02be063f3aa15adecb725342119c.tar.gz mitmproxy-0873566ff05c02be063f3aa15adecb725342119c.tar.bz2 mitmproxy-0873566ff05c02be063f3aa15adecb725342119c.zip |
Merge remote-tracking branch 'origin/master' into pr-3448
Diffstat (limited to 'mitmproxy/addons')
-rw-r--r-- | mitmproxy/addons/block.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/clientplayback.py | 5 | ||||
-rw-r--r-- | mitmproxy/addons/core.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/cut.py | 8 | ||||
-rw-r--r-- | mitmproxy/addons/eventstore.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/export.py | 22 | ||||
-rw-r--r-- | mitmproxy/addons/onboarding.py | 3 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/__init__.py | 37 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/app.py | 118 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/static/mitmproxy.css | 4 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/templates/frame.html | 4 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/templates/index.html | 10 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/templates/layout.html | 2 | ||||
-rw-r--r-- | mitmproxy/addons/script.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/serverplayback.py | 11 | ||||
-rw-r--r-- | mitmproxy/addons/session.py | 4 | ||||
-rw-r--r-- | mitmproxy/addons/stickycookie.py | 1 | ||||
-rw-r--r-- | mitmproxy/addons/view.py | 20 |
18 files changed, 98 insertions, 159 deletions
diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py index 91f9f709..4ccde0e1 100644 --- a/mitmproxy/addons/block.py +++ b/mitmproxy/addons/block.py @@ -36,4 +36,4 @@ class Block: layer.reply.kill() if ctx.options.block_global and address.is_global: ctx.log.warn("Client connection from %s killed by block_global" % astr) - layer.reply.kill()
\ No newline at end of file + layer.reply.kill() diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index c56c0e74..7bdaeb33 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -203,8 +203,9 @@ class ClientPlayback: # https://github.com/mitmproxy/mitmproxy/issues/2197 if hf.request.http_version == "HTTP/2.0": hf.request.http_version = "HTTP/1.1" - host = hf.request.headers.pop(":authority") - hf.request.headers.insert(0, "host", host) + host = hf.request.headers.pop(":authority", None) + if host is not None: + hf.request.headers.insert(0, "host", host) self.q.put(hf) ctx.master.addons.trigger("update", lst) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index a908dbb3..5c9bbcd0 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -289,7 +289,7 @@ class Core: """ The possible values for an encoding specification. """ - return ["gzip", "deflate", "br"] + return ["gzip", "deflate", "br", "zstd"] @command.command("options.load") def options_load(self, path: mitmproxy.types.Path) -> None: diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 6bb52e84..9aff2878 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -126,20 +126,18 @@ class Cut: format is UTF-8 encoded CSV. If there is exactly one row and one column, the data is written to file as-is, with raw bytes preserved. """ + v: typing.Union[str, bytes] fp = io.StringIO(newline="") if len(cuts) == 1 and len(flows) == 1: v = extract(cuts[0], flows[0]) - if isinstance(v, bytes): - fp.write(strutils.always_str(v)) - else: - fp.write(v) + fp.write(strutils.always_str(v)) # type: ignore ctx.log.alert("Clipped single cut.") else: writer = csv.writer(fp) for f in flows: vals = [extract(c, f) for c in cuts] writer.writerow( - [strutils.always_str(v) or "" for v in vals] # type: ignore + [strutils.always_str(v) for v in vals] ) ctx.log.alert("Clipped %s cuts as CSV." % len(cuts)) try: diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index 50fea7ab..188a3b39 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -14,7 +14,7 @@ class EventStore: self.sig_refresh = blinker.Signal() @property - def size(self) -> int: + def size(self) -> typing.Optional[int]: return self.data.maxlen def log(self, entry: LogEntry) -> None: diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index a7152ae4..20dac655 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -12,18 +12,25 @@ import mitmproxy.types import pyperclip -def raise_if_missing_request(f: flow.Flow) -> None: +def cleanup_request(f: flow.Flow): if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") + request = f.request.copy() # type: ignore + request.decode(strict=False) + # a bit of clean-up + if request.method == 'GET' and request.headers.get("content-length", None) == "0": + request.headers.pop('content-length') + request.headers.pop(':authority', None) + return request def curl_command(f: flow.Flow) -> str: - raise_if_missing_request(f) + request = cleanup_request(f) args = ["curl"] - request = f.request.copy() # type: ignore - request.decode(strict=False) for k, v in request.headers.items(multi=True): + args += ["--compressed "] if k == 'accept-encoding' else [] args += ["-H", "%s:%s" % (k, v)] + if request.method != "GET": args += ["-X", request.method] args.append(request.url) @@ -39,9 +46,7 @@ def curl_command(f: flow.Flow) -> str: def httpie_command(f: flow.Flow) -> str: - raise_if_missing_request(f) - request = f.request.copy() # type: ignore - request.decode(strict=False) + request = cleanup_request(f) args = ["http", request.method, request.url] for k, v in request.headers.items(multi=True): args.append("%s:%s" % (k, v)) @@ -58,8 +63,7 @@ def httpie_command(f: flow.Flow) -> str: def raw(f: flow.Flow) -> bytes: - raise_if_missing_request(f) - return assemble.assemble_request(f.request) # type: ignore + return assemble.assemble_request(cleanup_request(f)) # type: ignore formats = dict( diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 900acb08..94ca7c49 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -10,7 +10,7 @@ class Onboarding(wsgiapp.WSGIApp): name = "onboarding" def __init__(self): - super().__init__(app.Adapter(app.application), None, None) + super().__init__(app, None, None) def load(self, loader): loader.add_option( @@ -32,6 +32,7 @@ class Onboarding(wsgiapp.WSGIApp): def configure(self, updated): self.host = ctx.options.onboarding_host self.port = ctx.options.onboarding_port + app.config["CONFDIR"] = ctx.options.confdir def request(self, f): if ctx.options.onboarding: diff --git a/mitmproxy/addons/onboardingapp/__init__.py b/mitmproxy/addons/onboardingapp/__init__.py index e69de29b..722fed03 100644 --- a/mitmproxy/addons/onboardingapp/__init__.py +++ b/mitmproxy/addons/onboardingapp/__init__.py @@ -0,0 +1,37 @@ +import os + +from flask import Flask, render_template + +from mitmproxy.options import CONF_BASENAME, CONF_DIR + +app = Flask(__name__) +# will be overridden in the addon, setting this here so that the Flask app can be run standalone. +app.config["CONFDIR"] = CONF_DIR + + +@app.route('/') +def index(): + return render_template("index.html") + + +@app.route('/cert/pem') +def pem(): + return read_cert("pem", "application/x-x509-ca-cert") + + +@app.route('/cert/p12') +def p12(): + return read_cert("p12", "application/x-pkcs12") + + +def read_cert(ext, content_type): + filename = CONF_BASENAME + f"-ca-cert.{ext}" + p = os.path.join(app.config["CONFDIR"], filename) + p = os.path.expanduser(p) + with open(p, "rb") as f: + cert = f.read() + + return cert, { + "Content-Type": content_type, + "Content-Disposition": f"inline; filename={filename}", + } diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py deleted file mode 100644 index ab136778..00000000 --- a/mitmproxy/addons/onboardingapp/app.py +++ /dev/null @@ -1,118 +0,0 @@ -import os - -import tornado.template -import tornado.web -import tornado.wsgi - -from mitmproxy.utils import data -from mitmproxy.proxy import config - -loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates")) - - -class Adapter(tornado.wsgi.WSGIAdapter): - # Tornado doesn't make the WSGI environment available to pages, so this - # hideous monkey patch is the easiest way to get to the mitmproxy.master - # variable. - - def __init__(self, application): - self._application = application - - def application(self, request): - request.master = self.environ["mitmproxy.master"] - return self._application(request) - - def __call__(self, environ, start_response): - self.environ = environ - return tornado.wsgi.WSGIAdapter.__call__( - self, - environ, - start_response - ) - - -class Index(tornado.web.RequestHandler): - - def get(self): - t = loader.load("index.html") - self.write(t.generate()) - - -class PEM(tornado.web.RequestHandler): - - @property - def filename(self): - return config.CONF_BASENAME + "-ca-cert.pem" - - def head(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - content_length = os.path.getsize(p) - - self.set_header("Content-Type", "application/x-x509-ca-cert") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - self.set_header("Content-Length", content_length) - - def get(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - self.set_header("Content-Type", "application/x-x509-ca-cert") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - with open(p, "rb") as f: - self.write(f.read()) - - -class P12(tornado.web.RequestHandler): - - @property - def filename(self): - return config.CONF_BASENAME + "-ca-cert.p12" - - def head(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - content_length = os.path.getsize(p) - - self.set_header("Content-Type", "application/x-pkcs12") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - self.set_header("Content-Length", content_length) - - def get(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - self.set_header("Content-Type", "application/x-pkcs12") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - with open(p, "rb") as f: - self.write(f.read()) - - -application = tornado.web.Application( - [ - (r"/", Index), - (r"/cert/pem", PEM), - (r"/cert/p12", P12), - ( - r"/static/(.*)", - tornado.web.StaticFileHandler, - { - "path": data.pkg_data.path("addons/onboardingapp/static") - } - ), - ], - # debug=True -) diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css index 969bd62b..e654d56b 100644 --- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css +++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css @@ -15,7 +15,7 @@ height: 300px; } -.bigtitle>div { +.bigtitle > div { display: table-cell; vertical-align: middle; } @@ -31,7 +31,7 @@ section { .innerlink { text-decoration: none; - border-bottom:1px dotted; + border-bottom: 1px dotted; margin-bottom: 15px; } diff --git a/mitmproxy/addons/onboardingapp/templates/frame.html b/mitmproxy/addons/onboardingapp/templates/frame.html index f00e1a66..13003f3c 100644 --- a/mitmproxy/addons/onboardingapp/templates/frame.html +++ b/mitmproxy/addons/onboardingapp/templates/frame.html @@ -3,7 +3,7 @@ <div class="row"> <div class="span12"> {% block body %} - {% end %} + {% endblock %} </div> </div> -{% end %} +{% endblock %} diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index 38aa27ed..aa471668 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -135,19 +135,19 @@ function changeTo(device) { <h2 class="text-center"> Click to install your mitmproxy certificate </h2> <div id="certbank" class="row"> <div class="col-md-3"> - <a onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a> + <a target="_blank" onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a> <p>Apple</p> </div> <div class="col-md-3"> - <a onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a> + <a target="_blank" onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a> <p>Windows</p> </div> <div class="col-md-3"> - <a onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a> + <a target="_blank" onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a> <p>Android</p> </div> <div class="col-md-3"> - <a onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a> + <a target="_blank" onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a> <p>Other</p> </div> </div> @@ -167,4 +167,4 @@ function changeTo(device) { between mitmproxy installations. </div> -{% end %} +{% endblock %} diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html index f6e1b286..cea8373b 100644 --- a/mitmproxy/addons/onboardingapp/templates/layout.html +++ b/mitmproxy/addons/onboardingapp/templates/layout.html @@ -28,7 +28,7 @@ <div class="container"> {% block content %} - {% end %} + {% endblock %} </div> </body> diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index a39ce5ce..3b2568c9 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -16,7 +16,7 @@ from mitmproxy import ctx import mitmproxy.types as mtypes -def load_script(path: str) -> types.ModuleType: +def load_script(path: str) -> typing.Optional[types.ModuleType]: fullname = "__mitmproxy_script__.{}".format( os.path.splitext(os.path.basename(path))[0] ) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 51ba60b4..0818696f 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -68,6 +68,13 @@ class ServerPlayback: to replay. """ ) + loader.add_option( + "server_replay_ignore_port", bool, False, + """ + Ignore request's destination port while searching for a saved flow + to replay. + """ + ) @command.command("replay.server") def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: @@ -110,7 +117,7 @@ class ServerPlayback: _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) - key: typing.List[typing.Any] = [str(r.port), str(r.scheme), str(r.method), str(path)] + key: typing.List[typing.Any] = [str(r.scheme), str(r.method), str(path)] if not ctx.options.server_replay_ignore_content: if ctx.options.server_replay_ignore_payload_params and r.multipart_form: key.extend( @@ -129,6 +136,8 @@ class ServerPlayback: if not ctx.options.server_replay_ignore_host: key.append(r.host) + if not ctx.options.server_replay_ignore_port: + key.append(r.port) filtered = [] ignore_params = ctx.options.server_replay_ignore_params or [] diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py index f9073c3e..6636b500 100644 --- a/mitmproxy/addons/session.py +++ b/mitmproxy/addons/session.py @@ -215,8 +215,8 @@ class Session: def __init__(self): self.db_store: SessionDB = None self._hot_store: collections.OrderedDict = collections.OrderedDict() - self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str]]] = {} - self._view: typing.List[typing.Tuple[typing.Union[int, float, str], str]] = [] + self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str, None]]] = {} + self._view: typing.List[typing.Tuple[typing.Union[int, float, str, None], str]] = [] self.order: str = orders[0] self.filter = matchall self._flush_period: float = self._FP_DEFAULT diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index fd530aaa..1651c1f6 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -53,6 +53,7 @@ class StickyCookie: self.flt = None def response(self, flow: http.HTTPFlow): + assert flow.response if self.flt: for name, (value, attrs) in flow.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 8d27840f..da9d19f9 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -238,18 +238,24 @@ class View(collections.abc.Sequence): """ Set focus to the next flow. """ - idx = self.focus.index + 1 - if self.inbounds(idx): - self.focus.flow = self[idx] + if self.focus.index is not None: + idx = self.focus.index + 1 + if self.inbounds(idx): + self.focus.flow = self[idx] + else: + pass @command.command("view.focus.prev") def focus_prev(self) -> None: """ Set focus to the previous flow. """ - idx = self.focus.index - 1 - if self.inbounds(idx): - self.focus.flow = self[idx] + if self.focus.index is not None: + idx = self.focus.index - 1 + if self.inbounds(idx): + self.focus.flow = self[idx] + else: + pass # Order @command.command("view.order.options") @@ -584,7 +590,7 @@ class Focus: """ def __init__(self, v: View) -> None: self.view = v - self._flow: mitmproxy.flow.Flow = None + self._flow: typing.Optional[mitmproxy.flow.Flow] = None self.sig_change = blinker.Signal() if len(self.view): self.flow = self.view[0] |