Legibility

3 files changed, 128 insertions, 50 deletions

diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py
index e81c7640..3560f0bd 100644
--- a/libmproxy/protocol/http.py
+++ b/libmproxy/protocol/http.py
@@ -27,7 +27,8 @@ def get_line(fp):
     Get a line, possibly preceded by a blank.
     """
     line = fp.readline()
-    if line == "\r\n" or line == "\n":  # Possible leftover from previous message
+    if line == "\r\n" or line == "\n":
+        # Possible leftover from previous message
         line = fp.readline()
     if line == "":
         raise tcp.NetLibDisconnect()
@@ -241,25 +242,47 @@ class HTTPRequest(HTTPMessage):
         is content associated, but not present. CONTENT_MISSING evaluates
         to False to make checking for the presence of content natural.
 
-        form_in: The request form which mitmproxy has received. The following values are possible:
-                 - relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form)
-                 - absolute (GET http://example.com:80/index.html)
-                 - authority-form (CONNECT example.com:443)
-                 Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
+        form_in: The request form which mitmproxy has received. The following
+        values are possible:
 
-        form_out: The request form which mitmproxy has send out to the destination
+             - relative (GET /index.html, OPTIONS *) (covers origin form and
+               asterisk form)
+             - absolute (GET http://example.com:80/index.html)
+             - authority-form (CONNECT example.com:443)
+             Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
+
+        form_out: The request form which mitmproxy has send out to the
+        destination
 
         timestamp_start: Timestamp indicating when request transmission started
 
         timestamp_end: Timestamp indicating when request transmission ended
     """
 
-    def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers,
-                 content, timestamp_start=None, timestamp_end=None, form_out=None):
+    def __init__(
+        self,
+        form_in,
+        method,
+        scheme,
+        host,
+        port,
+        path,
+        httpversion,
+        headers,
+        content,
+        timestamp_start=None,
+        timestamp_end=None,
+        form_out=None
+    ):
         assert isinstance(headers, ODictCaseless) or not headers
-        HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start,
-                             timestamp_end)
-
+        HTTPMessage.__init__(
+            self,
+            httpversion,
+            headers,
+            content,
+            timestamp_start,
+            timestamp_end
+        )
         self.form_in = form_in
         self.method = method
         self.scheme = scheme
@@ -312,7 +335,8 @@ class HTTPRequest(HTTPMessage):
 
         request_line = get_line(rfile)
 
-        if hasattr(rfile, "first_byte_timestamp"):  # more accurate timestamp_start
+        if hasattr(rfile, "first_byte_timestamp"):
+            # more accurate timestamp_start
             timestamp_start = rfile.first_byte_timestamp
 
         request_line_parts = http.parse_init(request_line)
@@ -683,7 +707,9 @@ class HTTPResponse(HTTPMessage):
         return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format(
             code=self.code,
             msg=self.msg,
-            contenttype=self.headers.get_first("content-type", "unknown content type"),
+            contenttype=self.headers.get_first(
+                "content-type", "unknown content type"
+            ),
             size=size
         )
 
@@ -704,7 +730,8 @@ class HTTPResponse(HTTPMessage):
             body_size_limit,
             include_body=include_body)
 
-        if hasattr(rfile, "first_byte_timestamp"):  # more accurate timestamp_start
+        if hasattr(rfile, "first_byte_timestamp"):
+            # more accurate timestamp_start
             timestamp_start = rfile.first_byte_timestamp
 
         if include_body:
@@ -745,7 +772,11 @@ class HTTPResponse(HTTPMessage):
 
     def _assemble_head(self, preserve_transfer_encoding=False):
         return '%s\r\n%s\r\n' % (
-            self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding))
+            self._assemble_first_line(),
+            self._assemble_headers(
+                preserve_transfer_encoding=preserve_transfer_encoding
+            )
+        )
 
     def assemble(self):
         """
@@ -755,7 +786,10 @@ class HTTPResponse(HTTPMessage):
             Raises an Exception if the request cannot be assembled.
         """
         if self.content == CONTENT_MISSING:
-            raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING")
+            raise proxy.ProxyError(
+                502,
+                "Cannot assemble flow with CONTENT_MISSING"
+            )
         head = self._assemble_head()
         if self.content:
             return head + self.content
@@ -822,8 +856,9 @@ class HTTPResponse(HTTPMessage):
             pairs = [pair.partition("=") for pair in header.split(';')]
             cookie_name = pairs[0][0]  # the key of the first key/value pairs
             cookie_value = pairs[0][2]  # the value of the first key/value pairs
-            cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in
-                                 pairs[1:]}
+            cookie_parameters = {
+                key.strip().lower(): value.strip() for key, sep, value in pairs[1:]
+            }
             cookies.append((cookie_name, (cookie_value, cookie_parameters)))
         return dict(cookies)
 
@@ -856,7 +891,8 @@ class HTTPFlow(Flow):
         self.response = None
         """@type: HTTPResponse"""
 
-        self.intercepting = False  # FIXME: Should that rather be an attribute of Flow?
+        # FIXME: Should that rather be an attribute of Flow?
+        self.intercepting = False
 
     _stateobject_attributes = Flow._stateobject_attributes.copy()
     _stateobject_attributes.update(
@@ -944,7 +980,9 @@ class HTTPFlow(Flow):
 
 class HttpAuthenticationError(Exception):
     def __init__(self, auth_headers=None):
-        super(HttpAuthenticationError, self).__init__("Proxy Authentication Required")
+        super(HttpAuthenticationError, self).__init__(
+            "Proxy Authentication Required"
+        )
         self.headers = auth_headers
         self.code = 407
 
@@ -1114,7 +1152,12 @@ class HTTPHandler(ProtocolHandler):
 
     def handle_server_reconnect(self, state):
         if state["state"] == "connect":
-            send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False)
+            send_connect_request(
+                self.c.server_conn,
+                state["host"],
+                state["port"],
+                update_state=False
+            )
         else:  # pragma: nocover
             raise RuntimeError("Unknown State: %s" % state["state"])
 
@@ -1138,11 +1181,11 @@ class HTTPHandler(ProtocolHandler):
             self.c.log(message_debug, level="debug")
 
         if flow:
-            # TODO: no flows without request or with both request and response at the moment.
+            # TODO: no flows without request or with both request and response
+            # at the moment.
             if flow.request and not flow.response:
                 flow.error = Error(message or message_debug)
                 self.c.channel.ask("error", flow)
-
         try:
             code = getattr(error, "code", 502)
             headers = getattr(error, "headers", None)
@@ -1156,12 +1199,22 @@ class HTTPHandler(ProtocolHandler):
 
     def send_error(self, code, message, headers):
         response = http_status.RESPONSES.get(code, "Unknown")
-        html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \
-                       (code, response, message)
+        html_content = """
+            <html>
+                <head>
+                    <title>%d %s</title>
+                </head>
+                <body %s</body>
+            </html>
+        """ % (code, response, message)
         self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response))
-        self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version)
+        self.c.client_conn.wfile.write(
+            "Server: %s\r\n" % self.c.config.server_version
+        )
         self.c.client_conn.wfile.write("Content-type: text/html\r\n")
-        self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content))
+        self.c.client_conn.wfile.write(
+            "Content-Length: %d\r\n" % len(html_content)
+        )
         if headers:
             for key, value in headers.items():
                 self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value))
@@ -1201,11 +1254,15 @@ class HTTPHandler(ProtocolHandler):
         # Now we can process the request.
         if request.form_in == "authority":
             if self.c.client_conn.ssl_established:
-                raise http.HttpError(400, "Must not CONNECT on already encrypted connection")
+                raise http.HttpError(
+                    400,
+                    "Must not CONNECT on already encrypted connection"
+                )
 
             if self.c.config.mode == "regular":
                 self.c.set_server_address((request.host, request.port))
-                flow.server_conn = self.c.server_conn  # Update server_conn attribute on the flow
+                # Update server_conn attribute on the flow
+                flow.server_conn = self.c.server_conn
                 self.c.establish_server_connection()
                 self.c.client_conn.send(
                     'HTTP/1.1 200 Connection established\r\n' +
@@ -1217,7 +1274,9 @@ class HTTPHandler(ProtocolHandler):
             elif self.c.config.mode == "upstream":
                 return None
             else:
-                pass  # CONNECT should never occur if we don't expect absolute-form requests
+                # CONNECT should never occur if we don't expect absolute-form
+                # requests
+                pass
 
         elif request.form_in == self.expected_form_in:
 
@@ -1225,61 +1284,77 @@ class HTTPHandler(ProtocolHandler):
 
             if request.form_in == "absolute":
                 if request.scheme != "http":
-                    raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme)
+                    raise http.HttpError(
+                        400,
+                        "Invalid request scheme: %s" % request.scheme
+                    )
                 if self.c.config.mode == "regular":
-                    # Update info so that an inline script sees the correct value at flow.server_conn
+                    # Update info so that an inline script sees the correct
+                    # value at flow.server_conn
                     self.c.set_server_address((request.host, request.port))
                     flow.server_conn = self.c.server_conn
 
             return None
-
-        raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" %
-                             (self.expected_form_in, request.form_in))
+        raise http.HttpError(
+            400, "Invalid HTTP request form (expected: %s, got: %s)" % (
+                self.expected_form_in, request.form_in
+            )
+        )
 
     def process_server_address(self, flow):
         # Depending on the proxy mode, server handling is entirely different
-        # We provide a mostly unified API to the user, which needs to be unfiddled here
+        # We provide a mostly unified API to the user, which needs to be
+        # unfiddled here
         # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
         address = netlib.tcp.Address((flow.request.host, flow.request.port))
 
         ssl = (flow.request.scheme == "https")
 
         if self.c.config.mode == "upstream":
-
-            # The connection to the upstream proxy may have a state we may need to take into account.
+            # The connection to the upstream proxy may have a state we may need
+            # to take into account.
             connected_to = None
             for s in flow.server_conn.state:
                 if s[0] == "http" and s[1]["state"] == "connect":
                     connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
 
-            # We need to reconnect if the current flow either requires a (possibly impossible)
-            # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else.
+            # We need to reconnect if the current flow either requires a
+            # (possibly impossible) change to the connection state, e.g. the
+            # host has changed but we already CONNECTed somewhere else.
             needs_server_change = (
                 ssl != self.c.server_conn.ssl_established
                 or
-                (connected_to and address != connected_to)  # HTTP proxying is "stateless", CONNECT isn't.
+                # HTTP proxying is "stateless", CONNECT isn't.
+                (connected_to and address != connected_to)
             )
 
             if needs_server_change:
                 # force create new connection to the proxy server to reset state
                 self.live.change_server(self.c.server_conn.address, force=True)
                 if ssl:
-                    send_connect_request(self.c.server_conn, address.host, address.port)
+                    send_connect_request(
+                        self.c.server_conn,
+                        address.host,
+                        address.port
+                    )
                     self.c.establish_ssl(server=True)
         else:
-            # If we're not in upstream mode, we just want to update the host and possibly establish TLS.
-            self.live.change_server(address, ssl=ssl)  # this is a no op if the addresses match.
+            # If we're not in upstream mode, we just want to update the host and
+            # possibly establish TLS. This is a no op if the addresses match.
+            self.live.change_server(address, ssl=ssl)
 
         flow.server_conn = self.c.server_conn
 
     def send_response_to_client(self, flow):
         if not flow.response.stream:
             # no streaming:
-            # we already received the full response from the server and can send it to the client straight away.
+            # we already received the full response from the server and can send
+            # it to the client straight away.
             self.c.client_conn.send(flow.response.assemble())
         else:
             # streaming:
-            # First send the headers and then transfer the response incrementally:
+            # First send the headers and then transfer the response
+            # incrementally:
             h = flow.response._assemble_head(preserve_transfer_encoding=True)
             self.c.client_conn.send(h)
             for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
@@ -1293,7 +1368,8 @@ class HTTPHandler(ProtocolHandler):
 
     def check_close_connection(self, flow):
         """
-        Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
+            Checks if the connection should be closed depending on the HTTP
+            semantics. Returns True, if so.
         """
         close_connection = (
             http.connection_close(flow.request.httpversion, flow.request.headers) or
diff --git a/test/fuzzing/straight_stream b/test/fuzzing/straight_stream
index a716a085..99af212f 100644
--- a/test/fuzzing/straight_stream
+++ b/test/fuzzing/straight_stream
@@ -1,6 +1,6 @@
 
 mitmdump: $MITMDUMP -q --stream 1 
 pathod: $PATHOD -q
-#pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  ./straight_stream_patterns
-pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  /tmp/err
+pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  ./straight_stream_patterns
+#pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  /tmp/err
 
diff --git a/test/fuzzing/straight_stream_patterns b/test/fuzzing/straight_stream_patterns
index f5ae06f2..591bf20d 100644
--- a/test/fuzzing/straight_stream_patterns
+++ b/test/fuzzing/straight_stream_patterns
@@ -3,3 +3,5 @@ get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a'
 get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9'
 get:'http://localhost:9999/p/':s'200:b"foo"':ir,':'
 get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"'
+get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-'
+get:'http://localhost:9999/p/':s'200:b"foo"':dr