Merge branch 'http-models'

9 files changed, 61 insertions, 88 deletions

diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py
index 4a2310c9..48cb0f87 100644
--- a/libmproxy/console/common.py
+++ b/libmproxy/console/common.py
@@ -390,7 +390,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2,
         req_timestamp = f.request.timestamp_start,
         req_is_replay = f.request.is_replay,
         req_method = f.request.method,
-        req_url = f.request.pretty_url(hostheader=hostheader),
+        req_url = f.request.pretty_url if hostheader else f.request.url,
 
         err_msg = f.error.msg if f.error else None,
         resp_code = f.response.status_code if f.response else None,
diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py
index 5979bff3..5271db4f 100644
--- a/libmproxy/console/flowview.py
+++ b/libmproxy/console/flowview.py
@@ -168,7 +168,7 @@ class FlowView(tabs.Tabs):
             self.show()
 
     def content_view(self, viewmode, message):
-        if message.body == CONTENT_MISSING:
+        if message.content == CONTENT_MISSING:
             msg, body = "", [urwid.Text([("error", "[content missing]")])]
             return msg, body
         else:
@@ -193,13 +193,13 @@ class FlowView(tabs.Tabs):
 
         try:
             description, lines = contentviews.get_content_view(
-                viewmode, message.body, headers=message.headers
+                viewmode, message.content, headers=message.headers
             )
         except ContentViewException:
             s = "Content viewer failed: \n" + traceback.format_exc()
             signals.add_event(s, "error")
             description, lines = contentviews.get_content_view(
-                contentviews.get("Raw"), message.body, headers=message.headers
+                contentviews.get("Raw"), message.content, headers=message.headers
             )
             description = description.replace("Raw", "Couldn't parse: falling back to Raw")
 
diff --git a/libmproxy/dump.py b/libmproxy/dump.py
index 3915d4c8..d2b130f1 100644
--- a/libmproxy/dump.py
+++ b/libmproxy/dump.py
@@ -174,15 +174,15 @@ class DumpMaster(flow.FlowMaster):
             )
             self.echo(headers, indent=4)
         if self.o.flow_detail >= 3:
-            if message.body == CONTENT_MISSING:
+            if message.content == CONTENT_MISSING:
                 self.echo("(content missing)", indent=4)
-            elif message.body:
+            elif message.content:
                 self.echo("")
 
                 try:
                     type, lines = contentviews.get_content_view(
                         contentviews.get("Auto"),
-                        message.body,
+                        message.content,
                         headers=message.headers
                     )
                 except ContentViewException:
@@ -190,7 +190,7 @@ class DumpMaster(flow.FlowMaster):
                     self.add_event(s, "debug")
                     type, lines = contentviews.get_content_view(
                         contentviews.get("Raw"),
-                        message.body,
+                        message.content,
                         headers=message.headers
                     )
 
@@ -241,7 +241,11 @@ class DumpMaster(flow.FlowMaster):
             DELETE="red"
         ).get(method.upper(), "magenta")
         method = click.style(method, fg=method_color, bold=True)
-        url = click.style(flow.request.pretty_url(self.showhost), bold=True)
+        if self.showhost:
+            url = flow.request.pretty_url
+        else:
+            url = flow.request.url
+        url = click.style(url, bold=True)
 
         line = "{stickycookie}{client} {method} {url}".format(
             stickycookie=stickycookie,
@@ -266,7 +270,7 @@ class DumpMaster(flow.FlowMaster):
         elif 400 <= code < 600:
             code_color = "red"
         code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
-        msg = click.style(flow.response.msg, fg=code_color, bold=True)
+        reason = click.style(flow.response.reason, fg=code_color, bold=True)
 
         if flow.response.content == CONTENT_MISSING:
             size = "(content missing)"
@@ -276,11 +280,11 @@ class DumpMaster(flow.FlowMaster):
 
         arrows = click.style("<<", bold=True)
 
-        line = "{replay} {arrows} {code} {msg} {size}".format(
+        line = "{replay} {arrows} {code} {reason} {size}".format(
             replay=replay,
             arrows=arrows,
             code=code,
-            msg=msg,
+            reason=reason,
             size=size
         )
         self.echo(line)
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index 7d51b6d1..55a4dbcf 100644
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -252,7 +252,7 @@ class ServerPlaybackState:
         ]
 
         if not self.ignore_content:
-            form_contents = r.get_form()
+            form_contents = r.urlencoded_form or r.multipart_form
             if self.ignore_payload_params and form_contents:
                 key.extend(
                     p for p in form_contents
diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py
index 1815d6f5..08cb52bf 100644
--- a/libmproxy/models/http.py
+++ b/libmproxy/models/http.py
@@ -6,9 +6,8 @@ import time
 
 from libmproxy import utils
 from netlib import encoding
-from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING
+from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING, decoded
 from netlib.tcp import Address
-from netlib.utils import native
 from .. import version, stateobject
 from .flow import Flow
 
@@ -17,7 +16,6 @@ class MessageMixin(stateobject.StateObject):
     _stateobject_attributes = dict(
         http_version=bytes,
         headers=Headers,
-        body=bytes,
         timestamp_start=float,
         timestamp_end=float
     )
@@ -26,9 +24,9 @@ class MessageMixin(stateobject.StateObject):
     def get_state(self, short=False):
         ret = super(MessageMixin, self).get_state(short)
         if short:
-            if self.body:
-                ret["contentLength"] = len(self.body)
-            elif self.body == CONTENT_MISSING:
+            if self.content:
+                ret["contentLength"] = len(self.content)
+            elif self.content == CONTENT_MISSING:
                 ret["contentLength"] = None
             else:
                 ret["contentLength"] = 0
@@ -41,9 +39,9 @@ class MessageMixin(stateobject.StateObject):
             Doesn't change the message iteself or its headers.
         """
         ce = self.headers.get("content-encoding")
-        if not self.body or ce not in encoding.ENCODINGS:
-            return self.body
-        return encoding.decode(ce, self.body)
+        if not self.content or ce not in encoding.ENCODINGS:
+            return self.content
+        return encoding.decode(ce, self.content)
 
     def decode(self):
         """
@@ -54,12 +52,12 @@ class MessageMixin(stateobject.StateObject):
             Returns True if decoding succeeded, False otherwise.
         """
         ce = self.headers.get("content-encoding")
-        if not self.body or ce not in encoding.ENCODINGS:
+        if not self.content or ce not in encoding.ENCODINGS:
             return False
-        data = encoding.decode(ce, self.body)
+        data = encoding.decode(ce, self.content)
         if data is None:
             return False
-        self.body = data
+        self.content = data
         self.headers.pop("content-encoding", None)
         return True
 
@@ -69,11 +67,14 @@ class MessageMixin(stateobject.StateObject):
             or "identity".
         """
         # FIXME: Error if there's an existing encoding header?
-        self.body = encoding.encode(e, self.body)
+        self.content = encoding.encode(e, self.content)
         self.headers["content-encoding"] = e
 
     def copy(self):
         c = copy.copy(self)
+        if hasattr(self, "data"):  # FIXME remove condition
+            c.data = copy.copy(self.data)
+
         c.headers = self.headers.copy()
         return c
 
@@ -86,8 +87,8 @@ class MessageMixin(stateobject.StateObject):
             Returns the number of replacements made.
         """
         with decoded(self):
-            self.body, count = utils.safe_subn(
-                pattern, repl, self.body, *args, **kwargs
+            self.content, count = utils.safe_subn(
+                pattern, repl, self.content, *args, **kwargs
             )
         fields = []
         for name, value in self.headers.fields:
@@ -147,7 +148,7 @@ class HTTPRequest(MessageMixin, Request):
 
     def __init__(
             self,
-            form_in,
+            first_line_format,
             method,
             scheme,
             host,
@@ -155,14 +156,14 @@ class HTTPRequest(MessageMixin, Request):
             path,
             http_version,
             headers,
-            body,
+            content,
             timestamp_start=None,
             timestamp_end=None,
             form_out=None,
     ):
         Request.__init__(
             self,
-            form_in,
+            first_line_format,
             method,
             scheme,
             host,
@@ -170,11 +171,11 @@ class HTTPRequest(MessageMixin, Request):
             path,
             http_version,
             headers,
-            body,
+            content,
             timestamp_start,
             timestamp_end,
         )
-        self.form_out = form_out or form_in
+        self.form_out = form_out or first_line_format  # FIXME remove
 
         # Have this request's cookies been modified by sticky cookies or auth?
         self.stickycookie = False
@@ -185,7 +186,8 @@ class HTTPRequest(MessageMixin, Request):
 
     _stateobject_attributes = MessageMixin._stateobject_attributes.copy()
     _stateobject_attributes.update(
-        form_in=str,
+        content=bytes,
+        first_line_format=str,
         method=bytes,
         scheme=bytes,
         host=bytes,
@@ -225,7 +227,7 @@ class HTTPRequest(MessageMixin, Request):
     @classmethod
     def wrap(self, request):
         req = HTTPRequest(
-            form_in=request.form_in,
+            first_line_format=request.form_in,
             method=request.method,
             scheme=request.scheme,
             host=request.host,
@@ -233,7 +235,7 @@ class HTTPRequest(MessageMixin, Request):
             path=request.path,
             http_version=request.http_version,
             headers=request.headers,
-            body=request.body,
+            content=request.content,
             timestamp_start=request.timestamp_start,
             timestamp_end=request.timestamp_end,
             form_out=(request.form_out if hasattr(request, 'form_out') else None),
@@ -288,9 +290,9 @@ class HTTPResponse(MessageMixin, Response):
             self,
             http_version,
             status_code,
-            msg,
+            reason,
             headers,
-            body,
+            content,
             timestamp_start=None,
             timestamp_end=None,
     ):
@@ -298,9 +300,9 @@ class HTTPResponse(MessageMixin, Response):
             self,
             http_version,
             status_code,
-            msg,
+            reason,
             headers,
-            body,
+            content,
             timestamp_start=timestamp_start,
             timestamp_end=timestamp_end,
         )
@@ -311,6 +313,7 @@ class HTTPResponse(MessageMixin, Response):
 
     _stateobject_attributes = MessageMixin._stateobject_attributes.copy()
     _stateobject_attributes.update(
+        body=bytes,
         status_code=int,
         msg=bytes
     )
@@ -336,9 +339,9 @@ class HTTPResponse(MessageMixin, Response):
         resp = HTTPResponse(
             http_version=response.http_version,
             status_code=response.status_code,
-            msg=response.msg,
+            reason=response.reason,
             headers=response.headers,
-            body=response.body,
+            content=response.content,
             timestamp_start=response.timestamp_start,
             timestamp_end=response.timestamp_end,
         )
@@ -400,22 +403,20 @@ class HTTPResponse(MessageMixin, Response):
 class HTTPFlow(Flow):
     """
     A HTTPFlow is a collection of objects representing a single HTTP
-    transaction. The main attributes are:
+    transaction.
 
+    Attributes:
         request: HTTPRequest object
         response: HTTPResponse object
         error: Error object
         server_conn: ServerConnection object
         client_conn: ClientConnection object
+        intercepted: Is this flow currently being intercepted?
+        live: Does this flow have a live client connection?
 
     Note that it's possible for a Flow to have both a response and an error
     object. This might happen, for instance, when a response was received
     from the server, but there was an error sending it back to the client.
-
-    The following additional attributes are exposed:
-
-        intercepted: Is this flow currently being intercepted?
-        live: Does this flow have a live client connection?
     """
 
     def __init__(self, client_conn, server_conn, live=None):
@@ -485,36 +486,6 @@ class HTTPFlow(Flow):
         return c
 
 
-class decoded(object):
-    """
-    A context manager that decodes a request or response, and then
-    re-encodes it with the same encoding after execution of the block.
-
-    Example:
-    with decoded(request):
-        request.content = request.content.replace("foo", "bar")
-    """
-
-    def __init__(self, o):
-        self.o = o
-        ce = o.headers.get("content-encoding")
-        if ce:
-            ce = native(ce, "ascii", "ignore")
-        if ce in encoding.ENCODINGS:
-            self.ce = ce
-        else:
-            self.ce = None
-
-    def __enter__(self):
-        if self.ce:
-            if not self.o.decode():
-                self.ce = None
-
-    def __exit__(self, type, value, tb):
-        if self.ce:
-            self.o.encode(self.ce)
-
-
 def make_error_response(status_code, message, headers=None):
     response = status_codes.RESPONSES.get(status_code, "Unknown")
     body = """
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py
index 34bccaf8..50765e50 100644
--- a/libmproxy/protocol/http.py
+++ b/libmproxy/protocol/http.py
@@ -54,7 +54,7 @@ class _StreamingHttpLayer(_HttpLayer):
 
     def read_response(self, request):
         response = self.read_response_headers()
-        response.body = b"".join(
+        response.content = b"".join(
             self.read_response_body(request, response)
         )
         return response
@@ -66,10 +66,10 @@ class _StreamingHttpLayer(_HttpLayer):
         raise NotImplementedError()
 
     def send_response(self, response):
-        if response.body == CONTENT_MISSING:
+        if response.content == CONTENT_MISSING:
             raise HttpException("Cannot assemble flow with CONTENT_MISSING")
         self.send_response_headers(response)
-        self.send_response_body(response, [response.body])
+        self.send_response_body(response, [response.content])
 
 
 class Http1Layer(_StreamingHttpLayer):
diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py
index d144e081..9764fb22 100644
--- a/libmproxy/protocol/tls.py
+++ b/libmproxy/protocol/tls.py
@@ -7,7 +7,6 @@ from construct import ConstructError
 import six
 from netlib.exceptions import InvalidCertificateException, TcpException, TlsException
 
-from netlib.http import ALPN_PROTO_HTTP1
 from ..contrib.tls._constructs import ClientHello
 from ..exceptions import ProtocolException, TlsProtocolException, ClientHandshakeException
 from .base import Layer
@@ -367,8 +366,8 @@ class TlsLayer(Layer):
         """
 
         # This gets triggered if we haven't established an upstream connection yet.
-        default_alpn = ALPN_PROTO_HTTP1
-        # alpn_preference = ALPN_PROTO_H2
+        default_alpn = b'http/1.1'
+        # alpn_preference = b'h2'
 
         if self.alpn_for_client_connection in options:
             choice = bytes(self.alpn_for_client_connection)
diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py
index 307d0c4b..f62b0c8e 100644
--- a/libmproxy/proxy/root_context.py
+++ b/libmproxy/proxy/root_context.py
@@ -6,7 +6,6 @@ import six
 
 from libmproxy.exceptions import ProtocolException
 from netlib.exceptions import TcpException
-from netlib.http import ALPN_PROTO_H2, ALPN_PROTO_HTTP1
 from ..protocol import (
     RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin,
     UpstreamConnectLayer
@@ -85,9 +84,9 @@ class RootContext(object):
         # 5. Check for TLS ALPN (HTTP1/HTTP2)
         if isinstance(top_layer, TlsLayer):
             alpn = top_layer.client_conn.get_alpn_proto_negotiated()
-            if alpn == ALPN_PROTO_H2:
+            if alpn == b'h2':
                 return Http2Layer(top_layer, 'transparent')
-            if alpn == ALPN_PROTO_HTTP1:
+            if alpn == b'http/1.1':
                 return Http1Layer(top_layer, 'transparent')
 
         # 6. Check for raw tcp mode
diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py
index 5c80584d..58dc77e7 100644
--- a/libmproxy/web/app.py
+++ b/libmproxy/web/app.py
@@ -29,7 +29,7 @@ class RequestHandler(tornado.web.RequestHandler):
     def json(self):
         if not self.request.headers.get("Content-Type").startswith("application/json"):
             return None
-        return json.loads(self.request.body)
+        return json.loads(self.request.content)
 
     @property
     def state(self):