diff options
31 files changed, 745 insertions, 356 deletions
@@ -1,3 +1,33 @@ +24 July 2015: mitmproxy 0.13 + + * Upstream certificate validation. See the --verify-upstream-cert, + --upstream-trusted-cadir and --upstream-trusted-ca parameters. Thanks to + Kyle Morton (github.com/kyle-m) for his work on this. + + * Add HTTP transparent proxy mode. This uses the host headers from HTTP + traffic (rather than SNI and IP address information from the OS) to + implement perform transparent proxying. Thanks to github.com/ijiro123 for + this feature. + + * Add ~src and ~dst REGEX filters, allowing matching on source and + destination addresses in the form of <IP>:<Port> + + * mitmproxy console: change g/G keyboard shortcuts to match less. Thanks to + Jose Luis Honorato (github.com/jlhonora). + + * mitmproxy console: Flow marking and unmarking. Marked flows are not + deleted when the flow list is cleared. Thanks to Jake Drahos + (github.com/drahosj). + + * mitmproxy console: add marking of flows + + * Remove the certforward feature. It was added to allow exploitation of + #gotofail, which is no longer a common vulnerability. Permitting this + hugely increased the complexity of packaging and distributing mitmproxy. + + + + 3 June 2015: mitmproxy 0.12.1 * mitmproxy console: mouse interaction - scroll in the flow list, click on diff --git a/CONTRIBUTORS b/CONTRIBUTORS index fe22d9da..3d056fb8 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,56 +1,51 @@ - 1067 Aldo Cortesi - 542 Maximilian Hils - 76 Marcelo Glezer + 1112 Aldo Cortesi + 569 Maximilian Hils + 79 Marcelo Glezer + 28 Jim Shaver 18 Henrik Nordstrom 13 Thomas Roth 12 Pedro Worcel - 11 Stephen Altamirano + 11 Jake Drahos 11 Justus Wingert - 11 Jim Shaver + 11 Stephen Altamirano 10 András Veres-Szentkirályi 9 Legend Tang - 8 Rouli 8 Jason A. Novak + 8 Rouli 7 Alexis Hildebrandt - 5 Matthias Urlichs + 6 Thomas Kriechbaumer 5 Brad Peabody + 5 Matthias Urlichs 5 Tomaz Muraus 5 elitest - 4 root + 5 iroiro123 + 4 Bryan Bishop + 4 Marc Liyanage 4 Valtteri Virtanen 4 Wade 524 - 4 Bryan Bishop 4 Youhei Sakurai - 4 Marc Liyanage + 4 root 3 Chris Neasbitt - 3 Zack B - 3 Kyle Manna + 3 David Weinstein 3 Eli Shvartsman + 3 Kyle Manna + 3 Zack B + 2 Bennett Blodinger 2 Choongwoo Han - 2 Rob Wills - 2 israel - 2 Mark E. Haase + 2 Heikki Hannikainen 2 Jaime Soriano Pastor 2 Jim Lloyd - 2 Heikki Hannikainen 2 Krzysztof Bielicki - 2 Bennett Blodinger + 2 Mark E. Haase 2 Michael Frister + 2 Rob Wills 2 alts - 1 Yuangxuan Wang - 1 capt8bit - 1 davidpshaw - 1 deployable - 1 joebowbeer - 1 meeee - 1 michaeljau - 1 peralta - 1 phil plante - 1 sentient07 - 1 vzvu3k6k + 2 isra17 + 2 israel 1 Andy Smith 1 Dan Wilbraham 1 David Shaw + 1 Doug Lethin 1 Eric Entzel 1 Felix Wolfsteller 1 Gabriel Kirkpatrick @@ -61,6 +56,7 @@ 1 James Billingham 1 Jean Regisser 1 Kit Randel + 1 Kyle Morton 1 Lucas Cimon 1 Mathieu Mitchell 1 Michael Bisbjerg @@ -82,7 +78,20 @@ 1 Steven Van Acker 1 Suyash 1 Tarashish Mishra + 1 TearsDontFalls 1 Terry Long 1 Ulrich Petri 1 Vyacheslav Bakhmutov 1 Wade Catron + 1 Yuangxuan Wang + 1 capt8bit + 1 davidpshaw + 1 deployable + 1 jlhonora + 1 joebowbeer + 1 meeee + 1 michaeljau + 1 peralta + 1 phil plante + 1 sentient07 + 1 vzvu3k6k @@ -14,11 +14,14 @@ __mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP. __libmproxy__ is the library that mitmproxy and mitmdump are built on. Documentation, tutorials and distribution packages can be found on the -mitmproxy.org website: - +mitmproxy.org website: [mitmproxy.org](http://mitmproxy.org). -You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html). +Installation Instructions are available at [mitmproxy.org/doc/install.html](http://mitmproxy.org/doc/install.html). + +You can join our developer chat on Slack: +[](https://mitmproxy-slack.herokuapp.com/) + Features @@ -54,7 +57,7 @@ $ git clone https://github.com/mitmproxy/mitmproxy.git $ git clone https://github.com/mitmproxy/netlib.git $ git clone https://github.com/mitmproxy/pathod.git $ cd mitmproxy -$ ./dev +$ source ./dev ``` The *dev* script will create a virtualenv environment in a directory called diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html index f5de7482..7cd6af25 100644 --- a/doc-src/scripting/inlinescripts.html +++ b/doc-src/scripting/inlinescripts.html @@ -145,8 +145,9 @@ You can view the API documentation using pydoc (which is installed with Python b ## Running scripts in parallel -We have a single flow primitive, so when a script is handling something, other requests block. -While that's a very desirable behaviour under some circumstances, scripts can be run threaded by using the <code>libmproxy.script.concurrent</code> decorator. +We have a single flow primitive, so when a script is blocking, other requests are not processed. +While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the <code>libmproxy.script.concurrent</code> decorator. +If your script does not block, you should avoid the overhead of the decorator. $!example("examples/nonblocking.py")!$ diff --git a/libmproxy/console/flowlist.py b/libmproxy/console/flowlist.py index bb23df75..46cd0de1 100644 --- a/libmproxy/console/flowlist.py +++ b/libmproxy/console/flowlist.py @@ -50,9 +50,9 @@ class EventListBox(urwid.ListBox): self.master.clear_events() key = None elif key == "G": - self.set_focus(0) - elif key == "g": self.set_focus(len(self.master.eventlist) - 1) + elif key == "g": + self.set_focus(0) return urwid.ListBox.keypress(self, size, key) @@ -338,10 +338,10 @@ class FlowListBox(urwid.ListBox): self.master.clear_flows() elif key == "e": self.master.toggle_eventlog() - elif key == "G": + elif key == "g": self.master.state.set_focus(0) signals.flowlist_change.send(self) - elif key == "g": + elif key == "G": self.master.state.set_focus(self.master.state.flow_count()) signals.flowlist_change.send(self) elif key == "l": diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index b20e54e4..d32ce5b4 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -5,9 +5,11 @@ import re import os import urwid +from netlib import odict +from netlib.http import user_agents + from . import common, signals from .. import utils, filt, script -from netlib import http_uastrings, http_cookies, odict FOOTER = [ @@ -416,9 +418,9 @@ class GridEditor(urwid.WidgetWrap): res.append(i[0]) self.callback(self.data_out(res), *self.cb_args, **self.cb_kwargs) signals.pop_view_state.send(self) - elif key == "G": - self.walker.set_focus(0) elif key == "g": + self.walker.set_focus(0) + elif key == "G": self.walker.set_focus(len(self.walker.lst) - 1) elif key in ["h", "left"]: self.walker.left() @@ -516,7 +518,7 @@ class HeaderEditor(GridEditor): return text def set_user_agent(self, k): - ua = http_uastrings.get_by_shortcut(k) + ua = user_agents.get_by_shortcut(k) if ua: self.walker.add_value( [ @@ -529,7 +531,7 @@ class HeaderEditor(GridEditor): if key == "U": signals.status_prompt_onekey.send( prompt = "Add User-Agent header:", - keys = [(i[0], i[1]) for i in http_uastrings.UASTRINGS], + keys = [(i[0], i[1]) for i in user_agents.UASTRINGS], callback = self.set_user_agent, ) return True @@ -592,7 +594,7 @@ class SetHeadersEditor(GridEditor): return text def set_user_agent(self, k): - ua = http_uastrings.get_by_shortcut(k) + ua = user_agents.get_by_shortcut(k) if ua: self.walker.add_value( [ @@ -606,7 +608,7 @@ class SetHeadersEditor(GridEditor): if key == "U": signals.status_prompt_onekey.send( prompt = "Add User-Agent header:", - keys = [(i[0], i[1]) for i in http_uastrings.UASTRINGS], + keys = [(i[0], i[1]) for i in user_agents.UASTRINGS], callback = self.set_user_agent, ) return True diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index 4e81a566..ba87348d 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -28,7 +28,7 @@ class HelpView(urwid.ListBox): keys = [ ("j, k", "down, up"), ("h, l", "left, right (in some contexts)"), - ("g, G", "go to end, beginning"), + ("g, G", "go to beginning, end"), ("space", "page down"), ("pg up/down", "page up/down"), ("arrows", "up, down, left, right"), @@ -42,12 +42,12 @@ class HelpView(urwid.ListBox): text.append(urwid.Text([("head", "\n\nGlobal keys:\n")])) keys = [ - ("c", "client replay"), + ("c", "client replay of HTTP requests"), ("i", "set interception pattern"), ("o", "options"), ("q", "quit / return to previous page"), ("Q", "quit without confirm prompt"), - ("S", "server replay"), + ("S", "server replay of HTTP responses"), ] text.extend( common.format_keyvals(keys, key="key", val="text", indent=4) @@ -108,8 +108,8 @@ class HelpView(urwid.ListBox): return None elif key == "?": key = None - elif key == "G": - self.set_focus(0) elif key == "g": + self.set_focus(0) + elif key == "G": self.set_focus(len(self.body.contents)) return urwid.ListBox.keypress(self, size, key) diff --git a/libmproxy/console/searchable.py b/libmproxy/console/searchable.py index 627d595d..dea0ac7f 100644 --- a/libmproxy/console/searchable.py +++ b/libmproxy/console/searchable.py @@ -33,10 +33,10 @@ class Searchable(urwid.ListBox): self.find_next(False) elif key == "N": self.find_next(True) - elif key == "G": + elif key == "g": self.set_focus(0) self.walker._modified() - elif key == "g": + elif key == "G": self.set_focus(len(self.walker) - 1) self.walker._modified() else: diff --git a/libmproxy/console/window.py b/libmproxy/console/window.py index 8754ed57..69d5e242 100644 --- a/libmproxy/console/window.py +++ b/libmproxy/console/window.py @@ -23,7 +23,7 @@ class Window(urwid.Frame): if not k: if args[1] == "mouse drag": signals.status_message.send( - message = "Hold down alt or ctrl to select text.", + message = "Hold down shift, alt or ctrl to select text.", expire = 1 ) elif args[1] == "mouse press" and args[2] == 4: diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 1983586b..bd17a807 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -241,6 +241,19 @@ class FUrl(_Rex): def __call__(self, f): return re.search(self.expr, f.request.url) +class FSrc(_Rex): + code = "src" + help = "Match source address" + + def __call__(self, f): + return f.client_conn and re.search(self.expr, repr(f.client_conn.address)) + +class FDst(_Rex): + code = "dst" + help = "Match destination address" + + def __call__(self, f): + return f.server_conn and re.search(self.expr, repr(f.server_conn.address)) class _Int(_Action): def __init__(self, num): @@ -313,6 +326,8 @@ filt_rex = [ FRequestContentType, FResponseContentType, FContentType, + FSrc, + FDst, ] filt_int = [ FCode @@ -324,7 +339,7 @@ def _make(): # ones. parts = [] for klass in filt_unary: - f = pp.Literal("~%s" % klass.code) + f = pp.Literal("~%s" % klass.code) + pp.WordEnd() f.setParseAction(klass.make) parts.append(f) @@ -333,12 +348,12 @@ def _make(): pp.QuotedString("\"", escChar='\\') |\ pp.QuotedString("'", escChar='\\') for klass in filt_rex: - f = pp.Literal("~%s" % klass.code) + rex.copy() + f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy() f.setParseAction(klass.make) parts.append(f) for klass in filt_int: - f = pp.Literal("~%s" % klass.code) + pp.Word(pp.nums) + f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums) f.setParseAction(klass.make) parts.append(f) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 59312ceb..4b725ae5 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -158,7 +158,7 @@ class StreamLargeBodies(object): def run(self, flow, is_request): r = flow.request if is_request else flow.response code = flow.response.code if flow.response else None - expected_size = netlib.http.expected_http_body_size( + expected_size = netlib.http.http1.HTTP1Protocol.expected_http_body_size( r.headers, is_request, flow.request.method, code ) if not (0 <= expected_size <= self.max_size): @@ -663,9 +663,12 @@ class FlowMaster(controller.Master): for s in self.scripts[:]: self.unload_script(s) - def unload_script(self, script): - script.unload() - self.scripts.remove(script) + def unload_script(self, script_obj): + try: + script_obj.unload() + except script.ScriptError as e: + self.add_event("Script error:\n" + str(e), "error") + self.scripts.remove(script_obj) def load_script(self, command): """ @@ -678,16 +681,16 @@ class FlowMaster(controller.Master): return v.args[0] self.scripts.append(s) - def run_single_script_hook(self, script, name, *args, **kwargs): - if script and not self.pause_scripts: - ret = script.run(name, *args, **kwargs) - if not ret[0] and ret[1]: - e = "Script error:\n" + ret[1][1] - self.add_event(e, "error") + def _run_single_script_hook(self, script_obj, name, *args, **kwargs): + if script_obj and not self.pause_scripts: + try: + script_obj.run(name, *args, **kwargs) + except script.ScriptError as e: + self.add_event("Script error:\n" + str(e), "error") def run_script_hook(self, name, *args, **kwargs): - for script in self.scripts: - self.run_single_script_hook(script, name, *args, **kwargs) + for script_obj in self.scripts: + self._run_single_script_hook(script_obj, name, *args, **kwargs) def get_ignore_filter(self): return self.server.config.check_ignore.patterns diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 2bb1f528..f2ac5acc 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,14 +1,16 @@ from __future__ import absolute_import import Cookie +import copy +import threading +import time import urllib import urlparse -import time -import copy from email.utils import parsedate_tz, formatdate, mktime_tz -import threading -from netlib import http, tcp, http_status, http_cookies -import netlib.utils -from netlib import odict + +import netlib +from netlib import http, tcp, odict, utils +from netlib.http import cookies, http1 + from .tcp import TCPHandler from .primitives import KILL, ProtocolHandler, Flow, Error from ..proxy.connection import ServerConnection @@ -303,6 +305,10 @@ class HTTPRequest(HTTPMessage): is_replay=bool ) + @property + def body(self): + return self.content + @classmethod def from_state(cls, state): f = cls( @@ -354,11 +360,10 @@ class HTTPRequest(HTTPMessage): if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() - req = http.read_request( - rfile, + protocol = http1.HTTP1Protocol(rfile=rfile, wfile=wfile) + req = protocol.read_request( include_body = include_body, body_size_limit = body_size_limit, - wfile = wfile ) if hasattr(rfile, "first_byte_timestamp"): @@ -375,7 +380,7 @@ class HTTPRequest(HTTPMessage): req.path, req.httpversion, req.headers, - req.content, + req.body, timestamp_start, timestamp_end ) @@ -642,7 +647,7 @@ class HTTPRequest(HTTPMessage): """ ret = odict.ODict() for i in self.headers["cookie"]: - ret.extend(http_cookies.parse_cookie_header(i)) + ret.extend(cookies.parse_cookie_header(i)) return ret def set_cookies(self, odict): @@ -650,7 +655,7 @@ class HTTPRequest(HTTPMessage): Takes an netlib.odict.ODict object. Over-writes any existing Cookie headers. """ - v = http_cookies.format_cookie_header(odict) + v = cookies.format_cookie_header(odict) self.headers["Cookie"] = [v] def replace(self, pattern, repl, *args, **kwargs): @@ -724,6 +729,12 @@ class HTTPResponse(HTTPMessage): msg=str ) + + @property + def body(self): + return self.content + + @classmethod def from_state(cls, state): f = cls(None, None, None, None, None) @@ -760,11 +771,12 @@ class HTTPResponse(HTTPMessage): if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() - httpversion, code, msg, headers, content = http.read_response( - rfile, + protocol = http1.HTTP1Protocol(rfile=rfile) + resp = protocol.read_response( request_method, body_size_limit, - include_body=include_body) + include_body=include_body + ) if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start @@ -776,11 +788,11 @@ class HTTPResponse(HTTPMessage): timestamp_end = None return HTTPResponse( - httpversion, - code, - msg, - headers, - content, + resp.httpversion, + resp.status_code, + resp.msg, + resp.headers, + resp.body, timestamp_start, timestamp_end ) @@ -894,7 +906,7 @@ class HTTPResponse(HTTPMessage): """ ret = [] for header in self.headers["set-cookie"]: - v = http_cookies.parse_set_cookie_header(header) + v = http.cookies.parse_set_cookie_header(header) if v: name, value, attrs = v ret.append([name, [value, attrs]]) @@ -910,7 +922,7 @@ class HTTPResponse(HTTPMessage): values = [] for i in odict.lst: values.append( - http_cookies.format_set_cookie_header( + http.cookies.format_set_cookie_header( i[0], i[1][0], i[1][1] @@ -1044,7 +1056,8 @@ class HTTPHandler(ProtocolHandler): self.c.server_conn.send(request_raw) # Only get the headers at first... flow.response = HTTPResponse.from_stream( - self.c.server_conn.rfile, flow.request.method, + self.c.server_conn.rfile, + flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False ) @@ -1081,10 +1094,13 @@ class HTTPHandler(ProtocolHandler): if flow.response.stream: flow.response.content = CONTENT_MISSING else: - flow.response.content = http.read_http_body( - self.c.server_conn.rfile, flow.response.headers, + protocol = http1.HTTP1Protocol(rfile=self.c.server_conn.rfile) + flow.response.content = protocol.read_http_body( + flow.response.headers, self.c.config.body_size_limit, - flow.request.method, flow.response.code, False + flow.request.method, + flow.response.code, + False ) flow.response.timestamp_end = utils.timestamp() @@ -1231,7 +1247,7 @@ class HTTPHandler(ProtocolHandler): pass def send_error(self, code, message, headers): - response = http_status.RESPONSES.get(code, "Unknown") + response = http.status_codes.RESPONSES.get(code, "Unknown") html_content = """ <html> <head> @@ -1285,6 +1301,7 @@ class HTTPHandler(ProtocolHandler): if not request.host and flow.server_conn: request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port + # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: @@ -1363,7 +1380,7 @@ class HTTPHandler(ProtocolHandler): # We provide a mostly unified API to the user, which needs to be # unfiddled here # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) - address = netlib.tcp.Address((flow.request.host, flow.request.port)) + address = tcp.Address((flow.request.host, flow.request.port)) ssl = (flow.request.scheme == "https") @@ -1417,8 +1434,8 @@ class HTTPHandler(ProtocolHandler): h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - chunks = http.read_http_body_chunked( - self.c.server_conn.rfile, + protocol = http1.HTTP1Protocol(rfile=self.c.server_conn.rfile) + chunks = protocol.read_http_body_chunked( flow.response.headers, self.c.config.body_size_limit, flow.request.method, @@ -1440,15 +1457,18 @@ class HTTPHandler(ProtocolHandler): semantics. Returns True, if so. """ close_connection = ( - http.connection_close( + http1.HTTP1Protocol.connection_close( flow.request.httpversion, - flow.request.headers) or http.connection_close( + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( flow.response.httpversion, - flow.response.headers) or http.expected_http_body_size( + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( flow.response.headers, False, flow.request.method, - flow.response.code) == -1) + flow.response.code) == -1 + ) if close_connection: if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index c5306b4a..ec91a6e0 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -2,7 +2,11 @@ from __future__ import absolute_import import os import re from OpenSSL import SSL -from netlib import http_auth, certutils, tcp + +import netlib +from netlib import http, certutils, tcp +from netlib.http import authentication + from .. import utils, platform, version from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode @@ -103,7 +107,7 @@ class ProxyConfig: self.openssl_method_server = ssl_version_server else: self.openssl_method_server = tcp.SSL_VERSIONS[ssl_version_server] - + if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: @@ -164,18 +168,18 @@ def process_proxy_options(parser, options): return parser.error( "Invalid single-user specification. Please use the format username:password") username, password = options.auth_singleuser.split(':') - password_manager = http_auth.PassManSingleUser(username, password) + password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: - password_manager = http_auth.PassManNonAnon() + password_manager = authentication.PassManNonAnon() elif options.auth_htpasswd: try: - password_manager = http_auth.PassManHtpasswd( + password_manager = authentication.PassManHtpasswd( options.auth_htpasswd) except ValueError as v: return parser.error(v.message) - authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy") else: - authenticator = http_auth.NullProxyAuth(None) + authenticator = authentication.NullProxyAuth(None) certs = [] for i in options.certs: diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2711bd0e..e77439fb 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -167,12 +167,12 @@ class ConnectionHandler: self.channel.tell("serverdisconnect", self) self.server_conn = None - def set_server_address(self, address): + def set_server_address(self, addr): """ Sets a new server address with the given priority. Does not re-establish either connection or SSL handshake. """ - address = tcp.Address.wrap(address) + address = tcp.Address.wrap(addr) # Don't reconnect to the same destination. if self.server_conn and self.server_conn.address == address: @@ -309,15 +309,15 @@ class ConnectionHandler: self.client_conn.finish() def log(self, msg, level, subs=()): - msg = [ + full_msg = [ "%s:%s: %s" % (self.client_conn.address.host, self.client_conn.address.port, msg)] for i in subs: - msg.append(" -> " + i) - msg = "\n".join(msg) - self.channel.tell("log", Log(msg, level)) + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) def find_cert(self): host = self.server_conn.address.host diff --git a/libmproxy/script.py b/libmproxy/script.py index 46edb86b..e13f0e2b 100644 --- a/libmproxy/script.py +++ b/libmproxy/script.py @@ -3,7 +3,7 @@ import os import traceback import threading import shlex -from . import controller +import sys class ScriptError(Exception): @@ -55,21 +55,17 @@ class ScriptContext: class Script: """ - The instantiator should do something along this vein: - - s = Script(argv, master) - s.load() + Script object representing an inline script. """ def __init__(self, command, master): - self.command = command - self.argv = self.parse_command(command) + self.args = self.parse_command(command) self.ctx = ScriptContext(master) self.ns = None self.load() @classmethod - def parse_command(klass, command): + def parse_command(cls, command): if not command or not command.strip(): raise ScriptError("Empty script command.") if os.name == "nt": # Windows: escape all backslashes in the path. @@ -89,54 +85,66 @@ class Script: def load(self): """ - Loads a module. + Loads an inline script. + + Returns: + The return value of self.run("start", ...) - Raises ScriptError on failure, with argument equal to an error - message that may be a formatted traceback. + Raises: + ScriptError on failure """ + if self.ns is not None: + self.unload() ns = {} + script_dir = os.path.dirname(os.path.abspath(self.args[0])) + sys.path.append(script_dir) try: - execfile(self.argv[0], ns, ns) - except Exception as v: - raise ScriptError(traceback.format_exc(v)) + execfile(self.args[0], ns, ns) + except Exception as e: + # Python 3: use exception chaining, https://www.python.org/dev/peps/pep-3134/ + raise ScriptError(traceback.format_exc(e)) + sys.path.pop() self.ns = ns - r = self.run("start", self.argv) - if not r[0] and r[1]: - raise ScriptError(r[1][1]) + return self.run("start", self.args) def unload(self): - return self.run("done") + ret = self.run("done") + self.ns = None + return ret def run(self, name, *args, **kwargs): """ - Runs a plugin method. + Runs an inline script hook. Returns: + The return value of the method. + None, if the script does not provide the method. - (True, retval) on success. - (False, None) on nonexistent method. - (False, (exc, traceback string)) if there was an exception. + Raises: + ScriptError if there was an exception. """ f = self.ns.get(name) if f: try: - return (True, f(self.ctx, *args, **kwargs)) - except Exception as v: - return (False, (v, traceback.format_exc(v))) + return f(self.ctx, *args, **kwargs) + except Exception as e: + raise ScriptError(traceback.format_exc(e)) else: - return (False, None) + return None class ReplyProxy(object): - def __init__(self, original_reply): - self._ignore_calls = 1 - self.lock = threading.Lock() + def __init__(self, original_reply, script_thread): self.original_reply = original_reply + self.script_thread = script_thread + self._ignore_call = True + self.lock = threading.Lock() def __call__(self, *args, **kwargs): with self.lock: - if self._ignore_calls > 0: - self._ignore_calls -= 1 + if self._ignore_call: + self.script_thread.start() + self._ignore_call = False return self.original_reply(*args, **kwargs) @@ -145,16 +153,19 @@ class ReplyProxy(object): def _handle_concurrent_reply(fn, o, *args, **kwargs): - # Make first call to o.reply a no op - - reply_proxy = ReplyProxy(o.reply) - o.reply = reply_proxy + # Make first call to o.reply a no op and start the script thread. + # We must not start the script thread before, as this may lead to a nasty race condition + # where the script thread replies a different response before the normal reply, which then gets swallowed. def run(): fn(*args, **kwargs) # If the script did not call .reply(), we have to do it now. reply_proxy() - ScriptThread(target=run).start() + + script_thread = ScriptThread(target=run) + + reply_proxy = ReplyProxy(o.reply, script_thread) + o.reply = reply_proxy class ScriptThread(threading.Thread): @@ -171,6 +182,7 @@ def concurrent(fn): "clientdisconnect"): def _concurrent(ctx, obj): _handle_concurrent_reply(fn, obj, ctx, obj) + return _concurrent raise NotImplementedError( - "Concurrent decorator not supported for this method.") + "Concurrent decorator not supported for '%s' method." % fn.func_name) diff --git a/libmproxy/version.py b/libmproxy/version.py index 7836c849..0af60af5 100644 --- a/libmproxy/version.py +++ b/libmproxy/version.py @@ -1,4 +1,6 @@ -IVERSION = (0, 12, 2) +from __future__ import (absolute_import, print_function, division) + +IVERSION = (0, 13, 1) VERSION = ".".join(str(i) for i in IVERSION) MINORVERSION = ".".join(str(i) for i in IVERSION[:2]) NAME = "mitmproxy" diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py index 29ae9e7a..d6082ee2 100644 --- a/libmproxy/web/app.py +++ b/libmproxy/web/app.py @@ -81,7 +81,8 @@ class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): @classmethod def broadcast(cls, **kwargs): - message = json.dumps(kwargs) + message = json.dumps(kwargs, ensure_ascii=False) + for conn in cls.connections: try: conn.write_message(message) diff --git a/release/build.py b/release/build.py new file mode 100755 index 00000000..f45efc1c --- /dev/null +++ b/release/build.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python + +from __future__ import (absolute_import, print_function, division, unicode_literals) +from contextlib import contextmanager +from os.path import dirname, realpath, join, exists, normpath +import os +import shutil +import subprocess +import glob +import re +from shlex import split +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if os.name == "nt": + venv_bin = "Scripts" +else: + venv_bin = "bin" + +root_dir = join(dirname(realpath(__file__)), "..", "..") +mitmproxy_dir = join(root_dir, "mitmproxy") +dist_dir = join(mitmproxy_dir, "dist") +test_venv_dir = join(root_dir, "venv.mitmproxy-release") + +all_projects = ("netlib", "pathod", "mitmproxy") +tools = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathod", "pathoc"], + "netlib": [] +} +if os.name == "nt": + tools["mitmproxy"].remove("mitmproxy") +version_files = { + "mitmproxy": normpath(join(root_dir, "mitmproxy/libmproxy/version.py")), + "pathod": normpath(join(root_dir, "pathod/libpathod/version.py")), + "netlib": normpath(join(root_dir, "netlib/netlib/version.py")), +} + + +@contextmanager +def empty_pythonpath(): + """ + Make sure that the regular python installation is not on the python path, + which would give us access to modules installed outside of our virtualenv. + """ + pythonpath = os.environ["PYTHONPATH"] + os.environ["PYTHONPATH"] = "" + yield + os.environ["PYTHONPATH"] = pythonpath + + +@contextmanager +def chdir(path): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("contributors") +def contributors(): + """ + Update CONTRIBUTORS.md + """ + print("Updating CONTRIBUTORS.md...") + contributors_data = subprocess.check_output(split("git shortlog -n -s")) + with open(join(mitmproxy_dir, "CONTRIBUTORS"), "w") as f: + f.write(contributors_data) + + +@cli.command("docs") +def docs(): + """ + Render the docs + """ + print("Rendering the docs...") + subprocess.check_call([ + "cshape", + join(mitmproxy_dir, "doc-src"), + join(mitmproxy_dir, "doc") + ]) + + +@cli.command("set-version") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.argument('version') +def set_version(projects, version): + """ + Update version information + """ + print("Update versions...") + version = ", ".join(version.split(".")) + for project, version_file in version_files.items(): + if project not in projects: + continue + print("Update %s..." % version_file) + with open(version_file, "rb") as f: + content = f.read() + new_content = re.sub(r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content) + with open(version_file, "wb") as f: + f.write(new_content) + + +@cli.command("git") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.argument('args', nargs=-1, required=True) +def git(projects, args): + """ + Run a git command on every project + """ + args = ["git"] + list(args) + for project in projects: + print("%s> %s..." % (project, " ".join(args))) + subprocess.check_call( + args, + cwd=join(root_dir, project) + ) + + +@cli.command("sdist") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +def sdist(projects): + """ + Build a source distribution + """ + with empty_pythonpath(): + print("Building release...") + if exists(dist_dir): + shutil.rmtree(dist_dir) + for project in projects: + print("Creating %s source distribution..." % project) + subprocess.check_call( + ["python", "./setup.py", "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar"], + cwd=join(root_dir, project) + ) + + +@cli.command("test") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.pass_context +def test(ctx, projects): + """ + Test the source distribution + """ + if not exists(dist_dir): + ctx.invoke(sdist) + + with empty_pythonpath(): + print("Creating virtualenv for test install...") + if exists(test_venv_dir): + shutil.rmtree(test_venv_dir) + subprocess.check_call(["virtualenv", "-q", test_venv_dir]) + + pip = join(test_venv_dir, venv_bin, "pip") + with chdir(dist_dir): + for project in projects: + print("Installing %s..." % project) + dist = glob.glob("./%s*" % project)[0] + subprocess.check_call([pip, "install", "-q", dist]) + + print("Running binaries...") + for project in projects: + for tool in tools[project]: + tool = join(test_venv_dir, venv_bin, tool) + print(tool) + print(subprocess.check_output([tool, "--version"])) + + print("Virtualenv available for further testing:") + print("source %s" % normpath(join(test_venv_dir, venv_bin, "activate"))) + + +@cli.command("upload") +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +def upload_release(username, password, repository): + """ + Upload source distributions to PyPI + """ + print("Uploading distributions...") + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + "%s/*" % dist_dir + ]) + + +# TODO: Fully automate build process. +# This wizard is missing OSX builds and updating mitmproxy.org. +@cli.command("wizard") +@click.option('--version', prompt=True) +@click.option('--username', prompt="PyPI Username") +@click.password_option(confirmation_prompt=False, prompt="PyPI Password") +@click.option('--repository', default="pypi") +@click.option('--project', '-p', 'projects', multiple=True, type=click.Choice(all_projects), default=all_projects) +@click.pass_context +def wizard(ctx, version, username, password, repository, projects): + """ + Interactive Release Wizard + """ + for project in projects: + if subprocess.check_output(["git", "status", "--porcelain"], cwd=join(root_dir, project)): + raise RuntimeError("%s repository is not clean." % project) + + # Build test release + ctx.invoke(sdist, projects=projects) + ctx.invoke(test, projects=projects) + click.confirm("Please test the release now. Is it ok?", abort=True) + + # bump version, update docs and contributors + ctx.invoke(set_version, version=version, projects=projects) + ctx.invoke(docs) + ctx.invoke(contributors) + + # version bump commit + tag + ctx.invoke(git, args=["commit", "-a", "-m", "bump version"], projects=projects) + ctx.invoke(git, args=["tag", "v" + version], projects=projects) + ctx.invoke(git, args=["push"], projects=projects) + ctx.invoke(git, args=["push", "--tags"], projects=projects) + + # Re-invoke sdist with bumped version + ctx.invoke(sdist, projects=projects) + click.confirm("All good, can upload to PyPI?", abort=True) + ctx.invoke(upload_release, username=username, password=password, repository=repository) + click.echo("All done!") + + +if __name__ == "__main__": + cli() diff --git a/release/contributors b/release/contributors deleted file mode 100755 index a7518219..00000000 --- a/release/contributors +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -git shortlog -n -s diff --git a/release/release-checklist b/release/release-checklist deleted file mode 100644 index 84a7152e..00000000 --- a/release/release-checklist +++ /dev/null @@ -1,37 +0,0 @@ - -- Check the version number: - - mitmproxy/libmproxy/version.py - netlib/netlib/version.py - pathod/libpathod/version.py - -- Ensure that the website style assets have been compiled for production, and -synced to the docs. - -- Render the docs: - cshape doc-src doc - -- Run the test release, make sure the output is sensible - ./release/test-release - -- Build the OSX binaries - - Follow instructions in osxbinaries - - Move to download dir: - mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download - -- Build the sources for each project: - python ./setup.py sdist - mv ./dist/FILE ~/mitmproxy/www.mitmproxy.org/src/download - -- Tag with the version number, and do: - git push --tags - -- Upload to pypi for each project: - - python ./setup.py sdist upload - -- Now bump the version number to be ready for the next cycle: - - mitmproxy/libmproxy/version.py - netlib/netlib/version.py - pathod/libpathod/version.py diff --git a/release/release-checklist.md b/release/release-checklist.md new file mode 100644 index 00000000..e6d9ae1f --- /dev/null +++ b/release/release-checklist.md @@ -0,0 +1,55 @@ +# Release Checklist + +## Test + + - Create the source distributions, make sure the output is sensible: + `./release/build.py release` + All source distributions can be found in `./dist`. + + - Test the source distributions: + `./release/build.py test` + This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. + +## Release + + - Verify that repositories are in a clean state: + `./release/build.py git status` + + - Update the version number in `version.py` for all projects: + `./release/build.py set-version 0.13` + + - Ensure that the website style assets have been compiled for production, and synced to the docs. + + - Render the docs, update CONTRIBUTORS file: + `./release/build.py docs contributors` + + - Make version bump commit for all projects, tag and push it: + `./release/build.py git commit -am "bump version"` + `./release/build.py git tag v0.13` + `./release/build.py git push --tags` + + - Recreate the source distributions with updated version information: + `./release/build.py sdist` + + - Build the OSX binaries + - Follow instructions in osx-binaries + - Move to download dir: + `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download` + + - Move all source distributions from `./dist` to the server: + `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` + + - Upload distributions in `./dist` to PyPI: + `./release/build.py upload` + You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. + ([more info](https://tom-christie.github.io/articles/pypi/)) + + - Now bump the version number to be ready for the next cycle: + + **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? + We should probably just leave it as-is and only bump once we actually do the next release. + + Also, we need a release policy. I propose the following: + - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. + - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. + This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)`
\ No newline at end of file @@ -11,10 +11,7 @@ here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.txt'), encoding='utf-8') as f: long_description = f.read() -scripts = ["mitmdump", "mitmweb"] -if os.name != "nt": - scripts.append("mitmproxy") - +# Core dependencies deps = { "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), "pyasn1>0.1.2", @@ -25,7 +22,8 @@ deps = { "pyparsing>=1.5.2", "html2text>=2015.4.14" } -script_deps = { +# A script -> additional dependencies dict. +scripts = { "mitmproxy": { "urwid>=1.3", "lxml>=3.3.6", @@ -34,14 +32,31 @@ script_deps = { "mitmdump": set(), "mitmweb": set() } -for script in scripts: - deps.update(script_deps[script]) +# Developer dependencies +dev_deps = { + "mock>=1.0.1", + "nose>=1.3.0", + "nose-cov>=1.6", + "coveralls>=0.4.1", + "click>=4.1", + "twine>=1.5.0", + "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), + "countershape" +} +# Add *all* script dependencies to developer dependencies. +for script_deps in scripts.values(): + dev_deps.update(script_deps) + +# Remove mitmproxy for Windows support. if os.name == "nt": + del scripts["mitmproxy"] deps.add("pydivert>=0.0.7") # Transparent proxying on Windows -console_scripts = [ - "%s = libmproxy.main:%s" % (s, s) for s in scripts -] +# Add dependencies for available scripts as core dependencies. +for script_deps in scripts.values(): + deps.update(script_deps) + +console_scripts = ["%s = libmproxy.main:%s" % (s, s) for s in scripts.keys()] setup( name="mitmproxy", @@ -75,15 +90,7 @@ setup( 'console_scripts': console_scripts}, install_requires=list(deps), extras_require={ - 'dev': [ - "mock>=1.0.1", - "nose>=1.3.0", - "nose-cov>=1.6", - "coveralls>=0.4.1", - "pathod>=%s, <%s" % - (version.MINORVERSION, - version.NEXT_MINORVERSION), - "countershape"], + 'dev': list(dev_deps), 'contentviews': [ "pyamf>=0.6.1", "protobuf>=2.5.0", diff --git a/test/scripts/a.py b/test/scripts/a.py index 210fea78..d4272ac8 100644 --- a/test/scripts/a.py +++ b/test/scripts/a.py @@ -1,7 +1,4 @@ -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('--var', type=int) +from a_helper import parser var = 0 diff --git a/test/scripts/a_helper.py b/test/scripts/a_helper.py new file mode 100644 index 00000000..2eeed0d4 --- /dev/null +++ b/test/scripts/a_helper.py @@ -0,0 +1,4 @@ +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('--var', type=int)
\ No newline at end of file diff --git a/test/scripts/unloaderr.py b/test/scripts/unloaderr.py new file mode 100644 index 00000000..f3743107 --- /dev/null +++ b/test/scripts/unloaderr.py @@ -0,0 +1,2 @@ +def done(ctx): + raise RuntimeError()
\ No newline at end of file diff --git a/test/test_examples.py b/test/test_examples.py index e9bccd2e..dce257cf 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -11,7 +11,9 @@ def test_load_scripts(): tmaster = tservers.TestMaster(config.ProxyConfig()) for f in scripts: - if "har_extractor" in f or "flowwriter" in f: + if "har_extractor" in f: + continue + if "flowwriter" in f: f += " -" if "iframe_injector" in f: f += " foo" # one argument required @@ -22,7 +24,7 @@ def test_load_scripts(): try: s = script.Script(f, tmaster) # Loads the script file. except Exception as v: - if not "ImportError" in str(v): + if "ImportError" not in str(v): raise else: s.unload() diff --git a/test/test_filt.py b/test/test_filt.py index 3ad17dfe..bcdf6e4c 100644 --- a/test/test_filt.py +++ b/test/test_filt.py @@ -241,6 +241,23 @@ class TestMatching: assert self.q("~c 200", s) assert not self.q("~c 201", s) + def test_src(self): + q = self.req() + assert self.q("~src address", q) + assert not self.q("~src foobar", q) + assert self.q("~src :22", q) + assert not self.q("~src :99", q) + assert self.q("~src address:22", q) + + def test_dst(self): + q = self.req() + q.server_conn = tutils.tserver_conn() + assert self.q("~dst address", q) + assert not self.q("~dst foobar", q) + assert self.q("~dst :22", q) + assert not self.q("~dst :99", q) + assert self.q("~dst address:22", q) + def test_and(self): s = self.resp() assert self.q("~c 200 & ~h head", s) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index d8489d4d..747fdc1e 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -327,11 +327,11 @@ class TestInvalidRequests(tservers.HTTPProxTest): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) assert r.status_code == 400 - assert "Must not CONNECT on already encrypted connection" in r.content + assert "Must not CONNECT on already encrypted connection" in r.body def test_relative_request(self): p = self.pathoc_raw() p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid HTTP request form" in r.content + assert "Invalid HTTP request form" in r.body diff --git a/test/test_proxy.py b/test/test_proxy.py index 77051edd..01fbe953 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -31,7 +31,9 @@ class TestServerConnection: f.server_conn = sc f.request.path = "/p/200:da" sc.send(f.request.assemble()) - assert http.read_response(sc.rfile, f.request.method, 1000) + + protocol = http.http1.HTTP1Protocol(rfile=sc.rfile) + assert protocol.read_response(f.request.method, 1000) assert self.d.last_log() sc.finish() diff --git a/test/test_script.py b/test/test_script.py index 0a063740..1b0e5a5b 100644 --- a/test/test_script.py +++ b/test/test_script.py @@ -1,120 +1,124 @@ -from libmproxy import script, flow -import tutils -import shlex import os import time import mock +from libmproxy import script, flow +import tutils + + +def test_simple(): + s = flow.State() + fm = flow.FlowMaster(None, s) + sp = tutils.test_data.path("scripts/a.py") + p = script.Script("%s --var 40" % sp, fm) + + assert "here" in p.ns + assert p.run("here") == 41 + assert p.run("here") == 42 + + tutils.raises(script.ScriptError, p.run, "errargs") + + # Check reload + p.load() + assert p.run("here") == 41 + + +def test_duplicate_flow(): + s = flow.State() + fm = flow.FlowMaster(None, s) + fm.load_script(tutils.test_data.path("scripts/duplicate_flow.py")) + f = tutils.tflow() + fm.handle_request(f) + assert fm.state.flow_count() == 2 + assert not fm.state.view[0].request.is_replay + assert fm.state.view[1].request.is_replay + + +def test_err(): + s = flow.State() + fm = flow.FlowMaster(None, s) + + tutils.raises( + "not found", + script.Script, "nonexistent", fm + ) + + tutils.raises( + "not a file", + script.Script, tutils.test_data.path("scripts"), fm + ) + + tutils.raises( + script.ScriptError, + script.Script, tutils.test_data.path("scripts/syntaxerr.py"), fm + ) + + tutils.raises( + script.ScriptError, + script.Script, tutils.test_data.path("scripts/loaderr.py"), fm + ) + + scr = script.Script(tutils.test_data.path("scripts/unloaderr.py"), fm) + tutils.raises(script.ScriptError, scr.unload) -class TestScript: - def test_simple(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - sp = tutils.test_data.path("scripts/a.py") - p = script.Script("%s --var 40" % sp, fm) - - assert "here" in p.ns - assert p.run("here") == (True, 41) - assert p.run("here") == (True, 42) - - ret = p.run("errargs") - assert not ret[0] - assert len(ret[1]) == 2 - - # Check reload - p.load() - assert p.run("here") == (True, 41) - - def test_duplicate_flow(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("scripts/duplicate_flow.py")) - f = tutils.tflow() - fm.handle_request(f) - assert fm.state.flow_count() == 2 - assert not fm.state.view[0].request.is_replay - assert fm.state.view[1].request.is_replay - - def test_err(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - - tutils.raises( - "not found", - script.Script, "nonexistent", fm - ) - - tutils.raises( - "not a file", - script.Script, tutils.test_data.path("scripts"), fm - ) - - tutils.raises( - script.ScriptError, - script.Script, tutils.test_data.path("scripts/syntaxerr.py"), fm - ) - - tutils.raises( - script.ScriptError, - script.Script, tutils.test_data.path("scripts/loaderr.py"), fm - ) - - def test_concurrent(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("scripts/concurrent_decorator.py")) - - with mock.patch("libmproxy.controller.DummyReply.__call__") as m: - f1, f2 = tutils.tflow(), tutils.tflow() - t_start = time.time() - fm.handle_request(f1) - f1.reply() - fm.handle_request(f2) - f2.reply() - - # Two instantiations - assert m.call_count == 0 # No calls yet. - assert (time.time() - t_start) < 0.09 - - def test_concurrent2(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - s = script.Script( - tutils.test_data.path("scripts/concurrent_decorator.py"), - fm) - s.load() - m = mock.Mock() - - class Dummy: - def __init__(self): - self.response = self - self.error = self - self.reply = m +def test_concurrent(): + s = flow.State() + fm = flow.FlowMaster(None, s) + fm.load_script(tutils.test_data.path("scripts/concurrent_decorator.py")) + with mock.patch("libmproxy.controller.DummyReply.__call__") as m: + f1, f2 = tutils.tflow(), tutils.tflow() t_start = time.time() + fm.handle_request(f1) + f1.reply() + fm.handle_request(f2) + f2.reply() + + # Two instantiations + assert m.call_count == 0 # No calls yet. + assert (time.time() - t_start) < 0.09 + - for hook in ("clientconnect", - "serverconnect", - "response", - "error", - "clientconnect"): - d = Dummy() - assert s.run(hook, d)[0] - d.reply() - while (time.time() - t_start) < 20 and m.call_count <= 5: - if m.call_count == 5: - return - time.sleep(0.001) - assert False - - def test_concurrent_err(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - tutils.raises( - "decorator not supported for this method", - script.Script, - tutils.test_data.path("scripts/concurrent_decorator_err.py"), - fm) +def test_concurrent2(): + s = flow.State() + fm = flow.FlowMaster(None, s) + s = script.Script( + tutils.test_data.path("scripts/concurrent_decorator.py"), + fm) + s.load() + m = mock.Mock() + + class Dummy: + def __init__(self): + self.response = self + self.error = self + self.reply = m + + t_start = time.time() + + for hook in ("clientconnect", + "serverconnect", + "response", + "error", + "clientconnect"): + d = Dummy() + s.run(hook, d) + d.reply() + while (time.time() - t_start) < 20 and m.call_count <= 5: + if m.call_count == 5: + return + time.sleep(0.001) + assert False + + +def test_concurrent_err(): + s = flow.State() + fm = flow.FlowMaster(None, s) + tutils.raises( + "Concurrent decorator not supported for 'start' method", + script.Script, + tutils.test_data.path("scripts/concurrent_decorator_err.py"), + fm) def test_command_parsing(): @@ -122,4 +126,4 @@ def test_command_parsing(): fm = flow.FlowMaster(None, s) absfilepath = os.path.normcase(tutils.test_data.path("scripts/a.py")) s = script.Script(absfilepath, fm) - assert os.path.isfile(s.argv[0]) + assert os.path.isfile(s.args[0]) diff --git a/test/test_server.py b/test/test_server.py index 9df4ef82..066e628a 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,15 +1,17 @@ import socket import time -from libmproxy.proxy.config import HostMatcher -import libpathod -from netlib import tcp, http_auth, http, socks -from libpathod import pathoc, pathod +from OpenSSL import SSL + +from netlib import tcp, http, socks from netlib.certutils import SSLCert -import tutils -import tservers +from netlib.http import authentication +from libpathod import pathoc, pathod + +from libmproxy.proxy.config import HostMatcher from libmproxy.protocol import KILL, Error from libmproxy.protocol.http import CONTENT_MISSING -from OpenSSL import SSL +import tutils +import tservers """ Note that the choice of response code in these tests matters more than you @@ -295,8 +297,8 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): class TestHTTPAuth(tservers.HTTPProxTest): - authenticator = http_auth.BasicProxyAuth( - http_auth.PassManSingleUser( + authenticator = http.authentication.BasicProxyAuth( + http.authentication.PassManSingleUser( "test", "test"), "realm") @@ -310,8 +312,8 @@ class TestHTTPAuth(tservers.HTTPProxTest): h'%s'='%s' """ % ( self.server.port, - http_auth.BasicProxyAuth.AUTH_HEADER, - http.assemble_http_basic_auth("basic", "test", "test") + http.authentication.BasicProxyAuth.AUTH_HEADER, + authentication.assemble_http_basic_auth("basic", "test", "test") )) assert ret.status_code == 202 @@ -526,7 +528,7 @@ class TestHttps2Http(tservers.ReverseProxTest): """ Returns a connected Pathoc instance. """ - p = libpathod.pathoc.Pathoc( + p = pathoc.Pathoc( ("localhost", self.proxy.port), ssl=ssl, sni=sni, fp=None ) p.connect() @@ -765,22 +767,15 @@ class TestStreamRequest(tservers.HTTPProxTest): (self.server.urlbase, spec)) connection.send("\r\n") - httpversion, code, msg, headers, content = http.read_response( - fconn, "GET", None, include_body=False) + protocol = http.http1.HTTP1Protocol(rfile=fconn) + resp = protocol.read_response("GET", None, include_body=False) - assert headers["Transfer-Encoding"][0] == 'chunked' - assert code == 200 + assert resp.headers["Transfer-Encoding"][0] == 'chunked' + assert resp.status_code == 200 chunks = list( - content for _, - content, - _ in http.read_http_body_chunked( - fconn, - headers, - None, - "GET", - 200, - False)) + content for _, content, _ in protocol.read_http_body_chunked( + resp.headers, None, "GET", 200, False)) assert chunks == ["this", "isatest", ""] connection.close() |