diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/console/common.py | 17 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 5 | ||||
| -rw-r--r-- | libmproxy/dump.py | 7 | ||||
| -rw-r--r-- | libmproxy/flow.py | 5 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 79 | ||||
| -rw-r--r-- | libmproxy/protocol/http_wrappers.py | 142 | ||||
| -rw-r--r-- | libmproxy/utils.py | 12 |
7 files changed, 69 insertions, 198 deletions
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 90bccfe7..5ce2c0b7 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -4,10 +4,13 @@ import urwid import urwid.util import os +from netlib.http.semantics import CONTENT_MISSING +import netlib.utils + from .. import utils -from ..protocol.http import CONTENT_MISSING, decoded +from ..protocol.http import decoded from . import signals -import netlib.utils + try: import pyperclip @@ -135,7 +138,7 @@ def raw_format_flow(f, focus, extended, padding): ) else: req.append(fcol(">>" if focus else " ", "focus")) - + if f["marked"]: req.append(fcol(SYMBOL_MARK, "mark")) @@ -249,7 +252,7 @@ def copy_flow_format_data(part, scope, flow): return None, "Request content is missing" with decoded(flow.request): if part == "h": - data += flow.request.assemble() + data += flow.client_protocol.assemble(flow.request) elif part == "c": data += flow.request.content else: @@ -262,7 +265,7 @@ def copy_flow_format_data(part, scope, flow): return None, "Response content is missing" with decoded(flow.response): if part == "h": - data += flow.response.assemble() + data += flow.client_protocol.assemble(flow.response) elif part == "c": data += flow.response.content else: @@ -295,7 +298,7 @@ def copy_flow(part, scope, flow, master, state): toclip = "" try: toclip = data.decode('utf-8') - except (UnicodeDecodeError): + except (UnicodeDecodeError): toclip = data try: @@ -391,7 +394,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2, err_msg = f.error.msg if f.error else None, resp_code = f.response.code if f.response else None, - + marked = marked, ) if f.response: diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index c6c4c10d..1e0f0c17 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -2,11 +2,14 @@ from __future__ import absolute_import import os import sys import urwid + from netlib import odict +from netlib.http.semantics import CONTENT_MISSING + from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview from .. import utils, controller -from ..protocol.http import HTTPRequest, HTTPResponse, CONTENT_MISSING, decoded +from ..protocol.http import HTTPRequest, HTTPResponse, decoded class SearchError(Exception): diff --git a/libmproxy/dump.py b/libmproxy/dump.py index ee8c65a0..bf409803 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -2,7 +2,10 @@ from __future__ import absolute_import, print_function import json import sys import os + +from netlib.http.semantics import CONTENT_MISSING import netlib.utils + from . import flow, filt, utils from .protocol import http @@ -173,7 +176,7 @@ class DumpMaster(flow.FlowMaster): if self.o.flow_detail >= 2: print(self.indent(4, message.headers.format()), file=self.outfile) if self.o.flow_detail >= 3: - if message.content == http.CONTENT_MISSING: + if message.content == CONTENT_MISSING: print(self.indent(4, "(content missing)"), file=self.outfile) elif message.content: print("", file=self.outfile) @@ -210,7 +213,7 @@ class DumpMaster(flow.FlowMaster): self._print_message(f.request) if f.response: - if f.response.content == http.CONTENT_MISSING: + if f.response.content == CONTENT_MISSING: sz = "(content missing)" else: sz = netlib.utils.pretty_size(len(f.response.content)) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 4b725ae5..82a25461 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,8 +8,11 @@ import Cookie import cookielib import os import re + from netlib import odict, wsgi, tcp +from netlib.http.semantics import CONTENT_MISSING import netlib.http + from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app from .protocol import http, handle @@ -921,7 +924,7 @@ class FlowMaster(controller.Master): return "Can't replay live request." if f.intercepted: return "Can't replay while intercepting..." - if f.request.content == http.CONTENT_MISSING: + if f.request.content == CONTENT_MISSING: return "Can't replay request with missing content..." if f.request: f.backup() diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index e0deadd5..7f1aa78b 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils from netlib.http import cookies, http1 +from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler from .primitives import KILL, ProtocolHandler, Flow, Error @@ -20,7 +21,6 @@ from .http_wrappers import decoded, HTTPRequest, HTTPResponse HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" HDR_FORM_MULTIPART = "multipart/form-data" -CONTENT_MISSING = 0 class KillSignal(Exception): @@ -39,14 +39,14 @@ def send_connect_request(conn, host, port, update_state=True): odict.ODictCaseless(), "" ) - conn.send(upstream_request.assemble()) protocol = http.http1.HTTP1Protocol(conn) + conn.send(protocol.assemble(upstream_request)) resp = HTTPResponse.from_protocol(protocol, upstream_request.method) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + "connection with upstream proxy: \r\n" + - str(resp.assemble())) + repr(resp)) if update_state: conn.state.append(("http", { "state": "connect", @@ -174,16 +174,15 @@ class HTTPHandler(ProtocolHandler): def get_response_from_server(self, flow): self.c.establish_server_connection() - request_raw = flow.request.assemble() for attempt in (0, 1): try: - self.c.server_conn.send(request_raw) + flow.server_protocol = http.http1.HTTP1Protocol(self.c.server_conn) + self.c.server_conn.send(flow.server_protocol.assemble(flow.request)) # Only get the headers at first... - protocol = http.http1.HTTP1Protocol(self.c.server_conn) flow.response = HTTPResponse.from_protocol( - protocol, + flow.server_protocol, flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False @@ -221,8 +220,8 @@ class HTTPHandler(ProtocolHandler): if flow.response.stream: flow.response.content = CONTENT_MISSING else: - protocol = http1.HTTP1Protocol(self.c.server_conn) - flow.response.content = protocol.read_http_body( + flow.server_protocol = http1.HTTP1Protocol(self.c.server_conn) + flow.response.content = flow.server_protocol.read_http_body( flow.response.headers, self.c.config.body_size_limit, flow.request.method, @@ -235,9 +234,9 @@ class HTTPHandler(ProtocolHandler): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) try: try: - protocol = http.http1.HTTP1Protocol(self.c.client_conn) + flow.client_protocol = http.http1.HTTP1Protocol(self.c.client_conn) req = HTTPRequest.from_protocol( - protocol, + flow.client_protocol, body_size_limit=self.c.config.body_size_limit ) except tcp.NetLibError: @@ -247,7 +246,7 @@ class HTTPHandler(ProtocolHandler): self.c.log( "request", "debug", - [req._assemble_first_line(req.form_in)] + [repr(req)] ) ret = self.process_request(flow, req) if ret is not None: @@ -276,8 +275,10 @@ class HTTPHandler(ProtocolHandler): flow.server_conn = self.c.server_conn self.c.log( - "response", "debug", [ - flow.response._assemble_first_line()]) + "response", + "debug", + [repr(flow.response)] + ) response_reply = self.c.channel.ask("response", flow) if response_reply is None or response_reply == KILL: raise KillSignal() @@ -553,30 +554,33 @@ class HTTPHandler(ProtocolHandler): # no streaming: # we already received the full response from the server and can # send it to the client straight away. - self.c.client_conn.send(flow.response.assemble()) + self.c.client_conn.send(flow.client_protocol.assemble(flow.response)) else: + raise NotImplementedError("HTTP streaming is currently not supported.") + # TODO: implement it according to new protocols and messages + # streaming: # First send the headers and then transfer the response # incrementally: - h = flow.response._assemble_head(preserve_transfer_encoding=True) - self.c.client_conn.send(h) - - protocol = http1.HTTP1Protocol(rfile=self.c.server_conn.rfile) - chunks = protocol.read_http_body_chunked( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 - ) - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - for chunk in chunks: - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() - flow.response.timestamp_end = utils.timestamp() + # h = flow.response._assemble_head(preserve_transfer_encoding=True) + # self.c.client_conn.send(h) + # + # protocol = http1.HTTP1Protocol(rfile=self.c.server_conn.rfile) + # chunks = protocol.read_http_body_chunked( + # flow.response.headers, + # self.c.config.body_size_limit, + # flow.request.method, + # flow.response.code, + # False, + # 4096 + # ) + # if callable(flow.response.stream): + # chunks = flow.response.stream(chunks) + # for chunk in chunks: + # for part in chunk: + # self.c.client_conn.wfile.write(part) + # self.c.client_conn.wfile.flush() + # flow.response.timestamp_end = utils.timestamp() def check_close_connection(self, flow): """ @@ -726,12 +730,13 @@ class RequestReplayThread(threading.Thread): sni=self.flow.server_conn.sni ) r.form_out = "relative" - server.send(r.assemble()) + + server.send(self.flow.server_protocol.assemble(r)) self.flow.server_conn = server - protocol = http.http1.HTTP1Protocol(server) + self.flow.server_protocol = http.http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( - protocol, + self.flow.server_protocol, r.method, body_size_limit=self.config.body_size_limit, ) diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index 7d3e3706..18a355dc 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -108,17 +108,6 @@ class MessageMixin(stateobject.StateObject): self.body = encoding.encode(e, self.body) self.headers["content-encoding"] = [e] - def size(self, **kwargs): - """ - Size in bytes of a fully rendered message, including headers and - HTTP lead-in. - """ - hl = len(self._assemble_head(**kwargs)) - if self.body: - return hl + len(self.body) - else: - return hl - def copy(self): c = copy.copy(self) c.headers = self.headers.copy() @@ -139,30 +128,6 @@ class MessageMixin(stateobject.StateObject): c += self.headers.replace(pattern, repl, *args, **kwargs) return c - def _assemble_first_line(self): - """ - Returns the assembled request/response line - """ - raise NotImplementedError() # pragma: nocover - - def _assemble_headers(self): - """ - Returns the assembled headers - """ - raise NotImplementedError() # pragma: nocover - - def _assemble_head(self): - """ - Returns the assembled request/response line plus headers - """ - raise NotImplementedError() # pragma: nocover - - def assemble(self): - """ - Returns the assembled request/response - """ - raise NotImplementedError() # pragma: nocover - class HTTPRequest(MessageMixin, semantics.Request): """ @@ -286,7 +251,8 @@ class HTTPRequest(MessageMixin, semantics.Request): def __repr__(self): return "<HTTPRequest: {0}>".format( - self._assemble_first_line(self.form_in)[:-9] + # just for visualisation purposes we use HTTP/1 protocol here + http.http1.HTTP1Protocol._assemble_request_first_line(self)[:-9] ) @classmethod @@ -315,66 +281,6 @@ class HTTPRequest(MessageMixin, semantics.Request): req.timestamp_end, ) - def _assemble_first_line(self, form=None): - form = form or self.form_out - - if form == "relative": - request_line = '%s %s HTTP/%s.%s' % ( - self.method, self.path, self.httpversion[0], self.httpversion[1] - ) - elif form == "authority": - request_line = '%s %s:%s HTTP/%s.%s' % ( - self.method, self.host, self.port, self.httpversion[0], - self.httpversion[1] - ) - elif form == "absolute": - request_line = '%s %s://%s:%s%s HTTP/%s.%s' % ( - self.method, self.scheme, self.host, - self.port, self.path, self.httpversion[0], - self.httpversion[1] - ) - else: - raise http.HttpError(400, "Invalid request form") - return request_line - - def _assemble_headers(self): - headers = self.headers.copy() - for k in self._headers_to_strip_off: - del headers[k] - if 'host' not in headers and self.scheme and self.host and self.port: - headers["Host"] = [utils.hostport(self.scheme, - self.host, - self.port)] - - # If content is defined (i.e. not None or CONTENT_MISSING), we always - # add a content-length header. - if self.body or self.body == "": - headers["Content-Length"] = [str(len(self.body))] - - return headers.format() - - def _assemble_head(self, form=None): - return "%s\r\n%s\r\n" % ( - self._assemble_first_line(form), self._assemble_headers() - ) - - def assemble(self, form=None): - """ - Assembles the request for transmission to the server. We make some - modifications to make sure interception works properly. - - Raises an Exception if the request cannot be assembled. - """ - if self.body == CONTENT_MISSING: - raise proxy.ProxyError( - 502, - "Cannot assemble flow with CONTENT_MISSING" - ) - head = self._assemble_head(form) - if self.body: - return head + self.body - else: - return head def __hash__(self): return id(self) @@ -699,50 +605,6 @@ class HTTPResponse(MessageMixin, semantics.Response): resp.timestamp_end, ) - def _assemble_first_line(self): - return 'HTTP/%s.%s %s %s' % \ - (self.httpversion[0], self.httpversion[1], self.code, self.msg) - - def _assemble_headers(self, preserve_transfer_encoding=False): - headers = self.headers.copy() - for k in self._headers_to_strip_off: - del headers[k] - if not preserve_transfer_encoding: - del headers['Transfer-Encoding'] - - # If body is defined (i.e. not None or CONTENT_MISSING), we always - # add a content-length header. - if self.body or self.body == "": - headers["Content-Length"] = [str(len(self.body))] - - return headers.format() - - def _assemble_head(self, preserve_transfer_encoding=False): - return '%s\r\n%s\r\n' % ( - self._assemble_first_line(), - self._assemble_headers( - preserve_transfer_encoding=preserve_transfer_encoding - ) - ) - - def assemble(self): - """ - Assembles the response for transmission to the client. We make some - modifications to make sure interception works properly. - - Raises an Exception if the request cannot be assembled. - """ - if self.body == CONTENT_MISSING: - raise proxy.ProxyError( - 502, - "Cannot assemble flow with CONTENT_MISSING" - ) - head = self._assemble_head() - if self.body: - return head + self.body - else: - return head - def _refresh_cookie(self, c, delta): """ Takes a cookie string c and a time delta in seconds, and returns diff --git a/libmproxy/utils.py b/libmproxy/utils.py index a29a53f5..78f74767 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -8,6 +8,7 @@ import functools import cgi import json +import netlib.utils def timestamp(): """ @@ -195,21 +196,12 @@ def parse_content_type(c): return ts[0].lower(), ts[1].lower(), d -def hostport(scheme, host, port): - """ - Returns the host component, with a port specifcation if needed. - """ - if (port, scheme) in [(80, "http"), (443, "https")]: - return host - else: - return "%s:%s" % (host, port) - def unparse_url(scheme, host, port, path=""): """ Returns a URL string, constructed from the specified compnents. """ - return "%s://%s%s" % (scheme, hostport(scheme, host, port), path) + return "%s://%s%s" % (scheme, netlib.utils.hostport(scheme, host, port), path) def clean_hanging_newline(t): |