From 43c52054241d8743996e8fd5321db93ada37a7b1 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Tue, 10 May 2016 14:03:14 -0600
Subject: mimtdump: add basic support for tcp flows

---
 mitmproxy/flow.py               | 61 +++++++++++++++++++++++++++++++++++------
 mitmproxy/models/__init__.py    |  5 +++-
 mitmproxy/models/flow.py        |  9 ++++++
 mitmproxy/models/http.py        |  6 ----
 mitmproxy/models/tcp.py         | 50 +++++++++++++++++++++++++++++++++
 mitmproxy/protocol/rawtcp.py    | 61 +++++++++++++++--------------------------
 mitmproxy/proxy/root_context.py |  2 +-
 7 files changed, 139 insertions(+), 55 deletions(-)
 create mode 100644 mitmproxy/models/tcp.py

diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py
index ccedd1d4..1d05d4bb 100644
--- a/mitmproxy/flow.py
+++ b/mitmproxy/flow.py
@@ -3,7 +3,6 @@
 """
 from __future__ import absolute_import
 
-import traceback
 from abc import abstractmethod, ABCMeta
 import hashlib
 import sys
@@ -18,12 +17,13 @@ from typing import List, Optional, Set
 from netlib import wsgi, odict
 from netlib.exceptions import HttpException
 from netlib.http import Headers, http1, cookies
+from netlib.utils import clean_bin
 from . import controller, tnetstring, filt, script, version, flow_format_compat
 from .onboarding import app
 from .proxy.config import HostMatcher
 from .protocol.http_replay import RequestReplayThread
 from .exceptions import Kill, FlowReadException
-from .models import ClientConnection, ServerConnection, HTTPFlow, HTTPRequest, FLOW_TYPES
+from .models import ClientConnection, ServerConnection, HTTPFlow, HTTPRequest, FLOW_TYPES, TCPFlow
 from collections import defaultdict
 
 
@@ -900,6 +900,17 @@ class FlowMaster(controller.ServerMaster):
                 self.handle_response(f)
             if f.error:
                 self.handle_error(f)
+        elif isinstance(f, TCPFlow):
+            messages = f.messages
+            f.messages = []
+            f.reply = controller.DummyReply()
+            self.handle_tcp_open(f)
+            while messages:
+                f.messages.append(messages.pop(0))
+                self.handle_tcp_message(f)
+            if f.error:
+                self.handle_tcp_error(f)
+            self.handle_tcp_close(f)
         else:
             raise NotImplementedError()
 
@@ -1087,18 +1098,52 @@ class FlowMaster(controller.ServerMaster):
             self.add_event('"{}" reloaded.'.format(s.filename), 'info')
         return ok
 
-    def handle_tcp_message(self, m):
-        self.run_script_hook("tcp_message", m)
-        m.reply()
+    def handle_tcp_open(self, flow):
+        self.state.add_flow(flow)
+        self.run_script_hook("tcp_open", flow)
+        flow.reply()
+
+    def handle_tcp_message(self, flow):
+        self.run_script_hook("tcp_message", flow)
+        message = flow.messages[-1]
+        direction = "->" if message.from_client else "<-"
+        self.add_event("{client} {direction} tcp {direction} {server}".format(
+            client=repr(flow.client_conn.address),
+            server=repr(flow.server_conn.address),
+            direction=direction,
+        ), "info")
+        self.add_event(clean_bin(message.content), "debug")
+        flow.reply()
+
+    def handle_tcp_error(self, flow):
+        if self.stream:
+            self.stream.add(flow)
+        self.add_event("Error in TCP connection to {}: {}".format(
+            repr(flow.server_conn.address),
+            flow.error
+        ), "info")
+        self.run_script_hook("tcp_error", flow)
+        flow.reply()
+
+    def handle_tcp_close(self, flow):
+        self.state.delete_flow(flow)
+        if self.stream:
+            self.stream.add(flow)
+        self.run_script_hook("tcp_close", flow)
+        flow.reply()
 
     def shutdown(self):
         super(FlowMaster, self).shutdown()
 
         # Add all flows that are still active
         if self.stream:
-            for i in self.state.flows:
-                if not i.response:
-                    self.stream.add(i)
+            for flow in self.state.flows:
+                # FIXME: We actually need to keep track of which flows are still active.
+                if isinstance(flow, HTTPFlow) and not flow.response:
+                    self.stream.add(flow)
+                if isinstance(flow, TCPFlow):
+                    # (assuming mitmdump only, this must be still active)
+                    self.stream.add(flow)
             self.stop_stream()
 
         self.unload_scripts()
diff --git a/mitmproxy/models/__init__.py b/mitmproxy/models/__init__.py
index df86eff4..3d9d9dae 100644
--- a/mitmproxy/models/__init__.py
+++ b/mitmproxy/models/__init__.py
@@ -7,9 +7,11 @@ from .http import (
 from netlib.http import decoded
 from .connections import ClientConnection, ServerConnection
 from .flow import Flow, Error
+from .tcp import TCPFlow
 
 FLOW_TYPES = dict(
-    http=HTTPFlow
+    http=HTTPFlow,
+    tcp=TCPFlow,
 )
 
 __all__ = [
@@ -18,5 +20,6 @@ __all__ = [
     "make_connect_response", "expect_continue_response",
     "ClientConnection", "ServerConnection",
     "Flow", "Error",
+    "TCPFlow"
     "FLOW_TYPES"
 ]
diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py
index 594147ec..1019c9fb 100644
--- a/mitmproxy/models/flow.py
+++ b/mitmproxy/models/flow.py
@@ -40,6 +40,9 @@ class Error(stateobject.StateObject):
     def __str__(self):
         return self.msg
 
+    def __repr__(self):
+        return self.msg
+
     @classmethod
     def from_state(cls, state):
         # the default implementation assumes an empty constructor. Override
@@ -99,6 +102,12 @@ class Flow(stateobject.StateObject):
             self._backup = state.pop("backup")
         super(Flow, self).set_state(state)
 
+    @classmethod
+    def from_state(cls, state):
+        f = cls(None, None)
+        f.set_state(state)
+        return f
+
     def copy(self):
         f = copy.copy(self)
 
diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py
index 77a809cf..75ffbfd0 100644
--- a/mitmproxy/models/http.py
+++ b/mitmproxy/models/http.py
@@ -191,12 +191,6 @@ class HTTPFlow(Flow):
         response=HTTPResponse
     )
 
-    @classmethod
-    def from_state(cls, state):
-        f = cls(None, None)
-        f.set_state(state)
-        return f
-
     def __repr__(self):
         s = "<HTTPFlow"
         for a in ("request", "response", "error", "client_conn", "server_conn"):
diff --git a/mitmproxy/models/tcp.py b/mitmproxy/models/tcp.py
new file mode 100644
index 00000000..22fd1f0c
--- /dev/null
+++ b/mitmproxy/models/tcp.py
@@ -0,0 +1,50 @@
+import time
+from typing import List
+
+from netlib.utils import Serializable
+from .flow import Flow
+
+
+class TCPMessage(Serializable):
+    def __init__(self, from_client, content, timestamp=None):
+        self.content = content
+        self.from_client = from_client
+        if timestamp is None:
+            timestamp = time.time()
+        self.timestamp = timestamp
+
+    @classmethod
+    def from_state(cls, state):
+        return cls(*state)
+
+    def get_state(self):
+        return self.from_client, self.content, self.timestamp
+
+    def set_state(self, state):
+        self.from_client = state.pop("from_client")
+        self.content = state.pop("content")
+        self.timestamp = state.pop("timestamp")
+
+    def __repr__(self):
+        return "{direction} {content}".format(
+            direction="->" if self.from_client else "<-",
+            content=repr(self.content)
+        )
+
+
+class TCPFlow(Flow):
+    """
+    A SSHFlow is a simplified representation of an SSH session.
+    """
+
+    def __init__(self, client_conn, server_conn, live=None):
+        super(TCPFlow, self).__init__("tcp", client_conn, server_conn, live)
+        self.messages = []  # type: List[TCPMessage]
+
+    _stateobject_attributes = Flow._stateobject_attributes.copy()
+    _stateobject_attributes.update(
+        messages=List[TCPMessage]
+    )
+
+    def __repr__(self):
+        return "<TCPFlow ({} messages)>".format(len(self.messages))
diff --git a/mitmproxy/protocol/rawtcp.py b/mitmproxy/protocol/rawtcp.py
index 7d18025e..5f6fca75 100644
--- a/mitmproxy/protocol/rawtcp.py
+++ b/mitmproxy/protocol/rawtcp.py
@@ -9,29 +9,26 @@ from netlib.exceptions import TcpException
 from netlib.tcp import ssl_read_select
 from netlib.utils import clean_bin
 from ..exceptions import ProtocolException
-from .base import Layer
-
+from ..models import Error
+from ..models.tcp import TCPFlow, TCPMessage
 
-class TcpMessage(object):
-
-    def __init__(self, client_conn, server_conn, sender, receiver, message):
-        self.client_conn = client_conn
-        self.server_conn = server_conn
-        self.sender = sender
-        self.receiver = receiver
-        self.message = message
+from .base import Layer
 
 
 class RawTCPLayer(Layer):
     chunk_size = 4096
 
-    def __init__(self, ctx, logging=True):
-        self.logging = logging
+    def __init__(self, ctx, ignore=False):
+        self.ignore = ignore
         super(RawTCPLayer, self).__init__(ctx)
 
     def __call__(self):
         self.connect()
 
+        if not self.ignore:
+            flow = TCPFlow(self.client_conn, self.server_conn, self)
+            self.channel.ask("tcp_open", flow)
+
         buf = memoryview(bytearray(self.chunk_size))
 
         client = self.client_conn.connection
@@ -51,38 +48,24 @@ class RawTCPLayer(Layer):
                         if isinstance(conn, SSL.Connection):
                             # We can't half-close a connection, so we just close everything here.
                             # Sockets will be cleaned up on a higher level.
-                            return
+                            break
                         else:
                             dst.shutdown(socket.SHUT_WR)
 
                         if len(conns) == 0:
-                            return
+                            break
                         continue
 
-                    tcp_message = TcpMessage(
-                        self.client_conn, self.server_conn,
-                        self.client_conn if dst == server else self.server_conn,
-                        self.server_conn if dst == server else self.client_conn,
-                        buf[:size].tobytes())
-                    self.channel.ask("tcp_message", tcp_message)
-                    dst.sendall(tcp_message.message)
-
-                    if self.logging:
-                        # log messages are prepended with the client address,
-                        # hence the "weird" direction string.
-                        if dst == server:
-                            direction = "-> tcp -> {}".format(repr(self.server_conn.address))
-                        else:
-                            direction = "<- tcp <- {}".format(repr(self.server_conn.address))
-                        data = clean_bin(tcp_message.message)
-                        self.log(
-                            "{}\r\n{}".format(direction, data),
-                            "info"
-                        )
+                    tcp_message = TCPMessage(dst == server, buf[:size].tobytes())
+                    if not self.ignore:
+                        flow.messages.append(tcp_message)
+                        self.channel.ask("tcp_message", flow)
+                    dst.sendall(tcp_message.content)
 
         except (socket.error, TcpException, SSL.Error) as e:
-            six.reraise(
-                ProtocolException,
-                ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e))),
-                sys.exc_info()[2]
-            )
+            if not self.ignore:
+                flow.error = Error("TCP connection closed unexpectedly: {}".format(repr(e)))
+                self.channel.tell("tcp_error", flow)
+        finally:
+            if not self.ignore:
+                self.channel.tell("tcp_close", flow)
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index 9caae02a..21478dfd 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -65,7 +65,7 @@ class RootContext(object):
                 else:
                     ignore = self.config.check_ignore((client_hello.client_sni, 443))
             if ignore:
-                return RawTCPLayer(top_layer, logging=False)
+                return RawTCPLayer(top_layer, ignore=True)
 
         # 2. Always insert a TLS layer, even if there's neither client nor server tls.
         # An inline script may upgrade from http to https,
-- 
cgit v1.2.3


From acd51befbb89f28397f8e3e52ead5bfa11fdc93f Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Wed, 11 May 2016 11:13:57 -0600
Subject: minor fixes, adjust tests

---
 mitmproxy/models/tcp.py                     | 2 +-
 mitmproxy/protocol/rawtcp.py                | 4 ++--
 test/mitmproxy/scripts/tcp_stream_modify.py | 7 ++++---
 test/mitmproxy/test_server.py               | 8 ++++----
 test/mitmproxy/tservers.py                  | 5 ++---
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/mitmproxy/models/tcp.py b/mitmproxy/models/tcp.py
index 22fd1f0c..7e966b95 100644
--- a/mitmproxy/models/tcp.py
+++ b/mitmproxy/models/tcp.py
@@ -34,7 +34,7 @@ class TCPMessage(Serializable):
 
 class TCPFlow(Flow):
     """
-    A SSHFlow is a simplified representation of an SSH session.
+    A TCPFlow is a simplified representation of a TCP session.
     """
 
     def __init__(self, client_conn, server_conn, live=None):
diff --git a/mitmproxy/protocol/rawtcp.py b/mitmproxy/protocol/rawtcp.py
index 5f6fca75..1b546c40 100644
--- a/mitmproxy/protocol/rawtcp.py
+++ b/mitmproxy/protocol/rawtcp.py
@@ -48,12 +48,12 @@ class RawTCPLayer(Layer):
                         if isinstance(conn, SSL.Connection):
                             # We can't half-close a connection, so we just close everything here.
                             # Sockets will be cleaned up on a higher level.
-                            break
+                            return
                         else:
                             dst.shutdown(socket.SHUT_WR)
 
                         if len(conns) == 0:
-                            break
+                            return
                         continue
 
                     tcp_message = TCPMessage(dst == server, buf[:size].tobytes())
diff --git a/test/mitmproxy/scripts/tcp_stream_modify.py b/test/mitmproxy/scripts/tcp_stream_modify.py
index 93b0d5c8..d7953ef9 100644
--- a/test/mitmproxy/scripts/tcp_stream_modify.py
+++ b/test/mitmproxy/scripts/tcp_stream_modify.py
@@ -1,3 +1,4 @@
-def tcp_message(ctx, tm):
-    if tm.sender == tm.server_conn:
-        tm.message = tm.message.replace("foo", "bar")
+def tcp_message(ctx, flow):
+    message = flow.messages[-1]
+    if not message.from_client:
+        message.content = message.content.replace("foo", "bar")
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index 454736d4..0701d52b 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -14,7 +14,7 @@ from pathod import pathoc, pathod
 
 from mitmproxy.proxy.config import HostMatcher
 from mitmproxy.exceptions import Kill
-from mitmproxy.models import Error, HTTPResponse
+from mitmproxy.models import Error, HTTPResponse, HTTPFlow
 
 from . import tutils, tservers
 
@@ -177,9 +177,9 @@ class TcpMixin:
         assert n.status_code == 304
         assert i.status_code == 305
         assert i2.status_code == 306
-        assert any(f.response.status_code == 304 for f in self.master.state.flows)
-        assert not any(f.response.status_code == 305 for f in self.master.state.flows)
-        assert not any(f.response.status_code == 306 for f in self.master.state.flows)
+        assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, HTTPFlow))
+        assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, HTTPFlow))
+        assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, HTTPFlow))
 
         # Test that we get the original SSL cert
         if self.ssl:
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index 4fa519cc..c9d68cfd 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -50,9 +50,8 @@ class TestMaster(flow.FlowMaster):
     def clear_log(self):
         self.log = []
 
-    def handle_log(self, l):
-        self.log.append(l.msg)
-        l.reply()
+    def add_event(self, message, level=None):
+        self.log.append(message)
 
 
 class ProxyThread(threading.Thread):
-- 
cgit v1.2.3


From d3c30d9005e42a68cb3f5a5440f30f01f100cbec Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Wed, 11 May 2016 16:34:18 -0600
Subject: fix tests, don't double-add error'd flows

---
 mitmproxy/flow.py          | 2 --
 test/mitmproxy/tservers.py | 4 +++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py
index 1d05d4bb..77ed8747 100644
--- a/mitmproxy/flow.py
+++ b/mitmproxy/flow.py
@@ -1116,8 +1116,6 @@ class FlowMaster(controller.ServerMaster):
         flow.reply()
 
     def handle_tcp_error(self, flow):
-        if self.stream:
-            self.stream.add(flow)
         self.add_event("Error in TCP connection to {}: {}".format(
             repr(flow.server_conn.address),
             flow.error
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index c9d68cfd..a1e9c713 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -161,7 +161,9 @@ class HTTPProxyTest(ProxyTestBase):
             q = "get:'/p/%s'" % spec
         else:
             q = "get:'%s/p/%s'" % (self.server.urlbase, spec)
-        return p.request(q)
+        resp = p.request(q)
+        p.close()
+        return resp
 
     def app(self, page):
         if self.ssl:
-- 
cgit v1.2.3


From d38989fe7e4aade0b7ea6c8c7d0de80c2b28ec81 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Thu, 12 May 2016 14:29:57 -0600
Subject: tests: finish connections properly

---
 test/mitmproxy/tservers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index a1e9c713..57225ee1 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -162,6 +162,7 @@ class HTTPProxyTest(ProxyTestBase):
         else:
             q = "get:'%s/p/%s'" % (self.server.urlbase, spec)
         resp = p.request(q)
+        p.finish()
         p.close()
         return resp
 
-- 
cgit v1.2.3


From f96697646ca495ddc045ab18134b8f70052c86ff Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Thu, 19 May 2016 22:09:00 -0700
Subject: add FlowMaster.active_flows

---
 mitmproxy/dump.py          |  2 +-
 mitmproxy/flow.py          | 22 +++++++++++-----------
 test/mitmproxy/tservers.py |  5 +----
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py
index aae397cd..f1eabdb8 100644
--- a/mitmproxy/dump.py
+++ b/mitmproxy/dump.py
@@ -320,7 +320,6 @@ class DumpMaster(flow.FlowMaster):
             self.outfile.flush()
 
     def _process_flow(self, f):
-        self.state.delete_flow(f)
         if self.filt and not f.match(self.filt):
             return
 
@@ -328,6 +327,7 @@ class DumpMaster(flow.FlowMaster):
 
     def handle_request(self, f):
         flow.FlowMaster.handle_request(self, f)
+        self.state.delete_flow(f)
         if f:
             f.reply()
         return f
diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py
index 77ed8747..51ef7193 100644
--- a/mitmproxy/flow.py
+++ b/mitmproxy/flow.py
@@ -653,8 +653,9 @@ class FlowMaster(controller.ServerMaster):
         if server:
             self.add_server(server)
         self.state = state
-        self.server_playback = None
-        self.client_playback = None
+        self.active_flows = set()  # type: Set[Flow]
+        self.server_playback = None  # type: Optional[ServerPlaybackState]
+        self.client_playback = None  # type: Optional[ClientPlaybackState]
         self.kill_nonreplay = False
         self.scripts = []  # type: List[script.Script]
         self.pause_scripts = False
@@ -1033,6 +1034,7 @@ class FlowMaster(controller.ServerMaster):
                 return
         if f not in self.state.flows:  # don't add again on replay
             self.state.add_flow(f)
+        self.active_flows.add(f)
         self.replacehooks.run(f)
         self.setheaders.run(f)
         self.process_new_request(f)
@@ -1053,6 +1055,7 @@ class FlowMaster(controller.ServerMaster):
         return f
 
     def handle_response(self, f):
+        self.active_flows.discard(f)
         self.state.update_flow(f)
         self.replacehooks.run(f)
         self.setheaders.run(f)
@@ -1099,7 +1102,9 @@ class FlowMaster(controller.ServerMaster):
         return ok
 
     def handle_tcp_open(self, flow):
-        self.state.add_flow(flow)
+        # TODO: This would break mitmproxy currently.
+        # self.state.add_flow(flow)
+        self.active_flows.add(flow)
         self.run_script_hook("tcp_open", flow)
         flow.reply()
 
@@ -1124,7 +1129,7 @@ class FlowMaster(controller.ServerMaster):
         flow.reply()
 
     def handle_tcp_close(self, flow):
-        self.state.delete_flow(flow)
+        self.active_flows.discard(flow)
         if self.stream:
             self.stream.add(flow)
         self.run_script_hook("tcp_close", flow)
@@ -1135,13 +1140,8 @@ class FlowMaster(controller.ServerMaster):
 
         # Add all flows that are still active
         if self.stream:
-            for flow in self.state.flows:
-                # FIXME: We actually need to keep track of which flows are still active.
-                if isinstance(flow, HTTPFlow) and not flow.response:
-                    self.stream.add(flow)
-                if isinstance(flow, TCPFlow):
-                    # (assuming mitmdump only, this must be still active)
-                    self.stream.add(flow)
+            for flow in self.active_flows:
+                self.stream.add(flow)
             self.stop_stream()
 
         self.unload_scripts()
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index 57225ee1..c9d68cfd 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -161,10 +161,7 @@ class HTTPProxyTest(ProxyTestBase):
             q = "get:'/p/%s'" % spec
         else:
             q = "get:'%s/p/%s'" % (self.server.urlbase, spec)
-        resp = p.request(q)
-        p.finish()
-        p.close()
-        return resp
+        return p.request(q)
 
     def app(self, page):
         if self.ssl:
-- 
cgit v1.2.3


From a3946d2a2d61a79f8f973c35f8321a37df3b8575 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Fri, 20 May 2016 13:27:26 -0700
Subject: tests++

---
 test/mitmproxy/test_flow.py |  8 ++++++++
 test/mitmproxy/tutils.py    | 26 ++++++++++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index b9c6a2f6..5c1e6a68 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -680,6 +680,10 @@ class TestSerialize:
         for i in range(3):
             f = tutils.tflow(err=True)
             w.add(f)
+        f = tutils.ttcpflow()
+        w.add(f)
+        f = tutils.ttcpflow(err=True)
+        w.add(f)
 
         sio.seek(0)
         return flow.FlowReader(sio)
@@ -1205,6 +1209,10 @@ class TestError:
         e3 = e.copy()
         assert e3.get_state() == e.get_state()
 
+    def test_repr(self):
+        e = Error("yay")
+        assert repr(e)
+
 
 class TestClientConnection:
 
diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py
index d51ac185..270a3911 100644
--- a/test/mitmproxy/tutils.py
+++ b/test/mitmproxy/tutils.py
@@ -3,6 +3,8 @@ import shutil
 import tempfile
 import argparse
 import sys
+
+from mitmproxy.models.tcp import TCPMessage
 from six.moves import cStringIO as StringIO
 from contextlib import contextmanager
 
@@ -11,7 +13,7 @@ from unittest.case import SkipTest
 import netlib.tutils
 from mitmproxy import utils, controller
 from mitmproxy.models import (
-    ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow
+    ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow
 )
 
 
@@ -44,6 +46,26 @@ def skip_appveyor(fn):
         return fn
 
 
+def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None):
+    if client_conn is True:
+        client_conn = tclient_conn()
+    if server_conn is True:
+        server_conn = tserver_conn()
+    if messages is True:
+        messages = [
+            TCPMessage(True, b"hello"),
+            TCPMessage(False, b"it's me"),
+        ]
+    if err is True:
+        err = terr()
+
+    f = TCPFlow(client_conn, server_conn)
+    f.messages = messages
+    f.error = err
+    f.reply = controller.DummyReply()
+    return f
+
+
 def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None):
     """
     @type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection
@@ -51,7 +73,7 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None):
     @type req:         bool | None | mitmproxy.protocol.http.HTTPRequest
     @type resp:        bool | None | mitmproxy.protocol.http.HTTPResponse
     @type err:         bool | None | mitmproxy.protocol.primitives.Error
-    @return:           bool | None | mitmproxy.protocol.http.HTTPFlow
+    @return:           mitmproxy.protocol.http.HTTPFlow
     """
     if client_conn is True:
         client_conn = tclient_conn()
-- 
cgit v1.2.3


From 354b8f84dfbfcc7dfe69a6ad847aa95d64a30330 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Mon, 23 May 2016 11:47:16 -0700
Subject: minor fixes (#1165)

---
 mitmproxy/console/__init__.py   | 4 ++--
 mitmproxy/console/grideditor.py | 1 -
 mitmproxy/utils.py              | 4 ----
 3 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py
index e75aed86..1dd032be 100644
--- a/mitmproxy/console/__init__.py
+++ b/mitmproxy/console/__init__.py
@@ -58,7 +58,7 @@ class ConsoleState(flow.State):
         return f
 
     def set_limit(self, limit):
-        ret = flow.State.set_limit(self, limit)
+        ret = super(ConsoleState, self).set_limit(limit)
         self.set_focus(self.focus)
         return ret
 
@@ -102,7 +102,7 @@ class ConsoleState(flow.State):
             self.focus -= 1
         if self.focus < 0:
             self.focus = None
-        ret = flow.State.delete_flow(self, f)
+        ret = super(ConsoleState, self).delete_flow(f)
         self.set_focus(self.focus)
         return ret
 
diff --git a/mitmproxy/console/grideditor.py b/mitmproxy/console/grideditor.py
index 11ce7d02..ea26d966 100644
--- a/mitmproxy/console/grideditor.py
+++ b/mitmproxy/console/grideditor.py
@@ -5,7 +5,6 @@ import re
 import os
 import urwid
 
-from netlib import odict
 from netlib.http import user_agents, cookies
 
 from . import common, signals
diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py
index cda5bba6..e56ac473 100644
--- a/mitmproxy/utils.py
+++ b/mitmproxy/utils.py
@@ -1,11 +1,7 @@
 from __future__ import (absolute_import, print_function, division)
-import os
 import datetime
-import re
 import time
 import json
-import importlib
-import inspect
 
 import netlib.utils
 
-- 
cgit v1.2.3


From e44947e8c267bb0fe4ff07235f28a3d3da549872 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Tue, 24 May 2016 16:18:10 -0700
Subject: Update issue_template.md

---
 issue_template.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/issue_template.md b/issue_template.md
index 08d390e4..2392f8c6 100644
--- a/issue_template.md
+++ b/issue_template.md
@@ -1,3 +1,4 @@
+<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) -->
 ##### Steps to reproduce the problem:
 
 1. 
-- 
cgit v1.2.3


From b0c90cc64f5090e762ba424a5f7aa57a22470d73 Mon Sep 17 00:00:00 2001
From: Thomas Kriechbaumer <thomas@kriechbaumer.name>
Date: Wed, 25 May 2016 19:42:22 +0200
Subject: add a py3 venv in dev environment

This allows us to run e.g. tests in a different python version:
$ venv3/bin/py.test test/netlib/test_tcp.py
---
 .gitignore |  1 +
 dev.sh     | 30 +++++++++++++++++++++++-------
 2 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index 79555e82..bccb8f99 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 MANIFEST
 */tmp
 /venv
+/venv3
 *.py[cdo]
 *.swp
 *.swo
diff --git a/dev.sh b/dev.sh
index a9c27248..0d1a8d98 100755
--- a/dev.sh
+++ b/dev.sh
@@ -1,13 +1,29 @@
-#!/bin/bash
+#!/bin/sh
 set -e
-VENV=./venv
+
+VENV="./venv"
+VENV3="${VENV}3"
 
 python -m virtualenv $VENV --always-copy
 . $VENV/bin/activate
-pip install -U pip setuptools
-pip install -r requirements.txt
+pip install -q -U pip setuptools
+pip install -q -r requirements.txt
 
 echo ""
-echo "* Created virtualenv environment in $VENV."
-echo "* Installed all dependencies into the virtualenv."
-echo "* You can now activate the virtualenv: \`. $VENV/bin/activate\`"
+echo "* Virtualenv created in $VENV and all dependencies installed."
+echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`"
+
+if $(python --version 2>&1 | grep -q "Python 2.") && command -v python3 >/dev/null 2>&1; then
+	echo ""
+	echo ""
+	
+  python3 -m virtualenv "$VENV3" --always-copy
+	. "$VENV3/bin/activate"
+	pip install -q -U pip setuptools
+	pip install -q -r requirements.txt
+
+	echo ""
+	echo "* Virtualenv created in $VENV3 and all dependencies installed."
+  echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV3/bin/activate\`"
+fi
+
-- 
cgit v1.2.3


From f7e77d543bcca84f75e09440841aacb582881da2 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Thu, 26 May 2016 09:35:21 +1200
Subject: Refactor dev.sh

- Parameterise the Python version. All of these now work:
    ./dev.sh            # Use default Python version
    ./dev.sh 2.7        # Explicitly use 2.7 in venv2.7
    ./dev.sh 3.5        # Explicitly use 3.5 in venv3.5

This should also work for Travis, which has a weird setup where Pytho3 can be
3.4.
---
 .gitignore |  3 +--
 dev.sh     | 29 ++++++++---------------------
 web/README |  2 +-
 3 files changed, 10 insertions(+), 24 deletions(-)

diff --git a/.gitignore b/.gitignore
index bccb8f99..1b44bd29 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,7 @@
 .DS_Store
 MANIFEST
 */tmp
-/venv
-/venv3
+/venv*
 *.py[cdo]
 *.swp
 *.swo
diff --git a/dev.sh b/dev.sh
index 0d1a8d98..111f09bc 100755
--- a/dev.sh
+++ b/dev.sh
@@ -1,29 +1,16 @@
 #!/bin/sh
 set -e
 
-VENV="./venv"
-VENV3="${VENV}3"
+PYVERSION=$1
+VENV="venv$1"
 
-python -m virtualenv $VENV --always-copy
-. $VENV/bin/activate
-pip install -q -U pip setuptools
-pip install -q -r requirements.txt
+echo "Creating dev environment in $VENV using Python $PYVERSION"
+
+python$PYVERSION -m virtualenv "$VENV" --always-copy
+. "$VENV/bin/activate"
+pip$PYVERSION install -q -U pip setuptools
+pip$PYVERSION install -q -r requirements.txt
 
 echo ""
 echo "* Virtualenv created in $VENV and all dependencies installed."
 echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`"
-
-if $(python --version 2>&1 | grep -q "Python 2.") && command -v python3 >/dev/null 2>&1; then
-	echo ""
-	echo ""
-	
-  python3 -m virtualenv "$VENV3" --always-copy
-	. "$VENV3/bin/activate"
-	pip install -q -U pip setuptools
-	pip install -q -r requirements.txt
-
-	echo ""
-	echo "* Virtualenv created in $VENV3 and all dependencies installed."
-  echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV3/bin/activate\`"
-fi
-
diff --git a/web/README b/web/README
index 63c3e6e0..c8e60379 100644
--- a/web/README
+++ b/web/README
@@ -3,4 +3,4 @@ Starting up
 
 - npm install
 - gulp
-- run mitmweb and open http://localhost:8081/
\ No newline at end of file
+- run mitmweb and open http://localhost:8081/
-- 
cgit v1.2.3


From 7a8da48a306dfc8e43239d7f2a141c465e40ab77 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Wed, 25 May 2016 19:16:02 -0700
Subject: escaped_str_to_bytes: support unicode on python 2

---
 netlib/utils.py           | 6 +++++-
 test/netlib/test_utils.py | 9 ++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/netlib/utils.py b/netlib/utils.py
index 7499f71f..fe11cb5b 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -439,10 +439,14 @@ def escaped_str_to_bytes(data):
     """
     Take an escaped string and return the unescaped bytes equivalent.
     """
-    if not isinstance(data, str):
+    if not isinstance(data, six.string_types):
+        if six.PY2:
+            raise ValueError("data must be str or unicode")
         raise ValueError("data must be str")
 
     if six.PY2:
+        if isinstance(data, unicode):
+            data = data.encode("utf8")
         return data.decode("string-escape")
 
     # This one is difficult - we use an undocumented Python API here
diff --git a/test/netlib/test_utils.py b/test/netlib/test_utils.py
index 1d8f7b0f..671ae66c 100644
--- a/test/netlib/test_utils.py
+++ b/test/netlib/test_utils.py
@@ -182,6 +182,9 @@ def test_bytes_to_escaped_str():
 
 def test_escaped_str_to_bytes():
     assert utils.escaped_str_to_bytes("foo") == b"foo"
-    assert utils.escaped_str_to_bytes(r"\x08") == b"\b"
-    assert utils.escaped_str_to_bytes(r"&!?=\\)") == br"&!?=\)"
-    assert utils.escaped_str_to_bytes(r"ü") == b'\xc3\xbc'
+    assert utils.escaped_str_to_bytes("\x08") == b"\b"
+    assert utils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)"
+    assert utils.escaped_str_to_bytes("ü") == b'\xc3\xbc'
+    assert utils.escaped_str_to_bytes(u"\\x08") == b"\b"
+    assert utils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)"
+    assert utils.escaped_str_to_bytes(u"ü") == b'\xc3\xbc'
\ No newline at end of file
-- 
cgit v1.2.3


From d3477e27fa4ffbcfa9c1b9aa937d1d54448cc597 Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Wed, 25 May 2016 20:11:34 -0700
Subject: bytes_to_escaped_str: always escape single quotes

---
 netlib/utils.py                   | 8 +++++++-
 test/netlib/test_utils.py         | 2 ++
 test/pathod/test_language_base.py | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/netlib/utils.py b/netlib/utils.py
index fe11cb5b..73b2adb3 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -425,6 +425,10 @@ def safe_subn(pattern, repl, target, *args, **kwargs):
 def bytes_to_escaped_str(data):
     """
     Take bytes and return a safe string that can be displayed to the user.
+
+    Single quotes are always escaped, double quotes are never escaped:
+        "'" + bytes_to_escaped_str(...) + "'"
+    gives a valid Python string.
     """
     # TODO: We may want to support multi-byte characters without escaping them.
     # One way to do would be calling .decode("utf8", "backslashreplace") first
@@ -432,7 +436,9 @@ def bytes_to_escaped_str(data):
 
     if not isinstance(data, bytes):
         raise ValueError("data must be bytes")
-    return repr(data).lstrip("b")[1:-1]
+    # We always insert a double-quote here so that we get a single-quoted string back
+    # https://stackoverflow.com/questions/29019340/why-does-python-use-different-quotes-for-representing-strings-depending-on-their
+    return repr('"' + data).lstrip("b")[2:-1]
 
 
 def escaped_str_to_bytes(data):
diff --git a/test/netlib/test_utils.py b/test/netlib/test_utils.py
index 671ae66c..fce1d0a7 100644
--- a/test/netlib/test_utils.py
+++ b/test/netlib/test_utils.py
@@ -178,6 +178,8 @@ def test_bytes_to_escaped_str():
     assert utils.bytes_to_escaped_str(b"\b") == r"\x08"
     assert utils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)"
     assert utils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc"
+    assert utils.bytes_to_escaped_str(b"'") == r"\'"
+    assert utils.bytes_to_escaped_str(b'"') == r'"'
 
 
 def test_escaped_str_to_bytes():
diff --git a/test/pathod/test_language_base.py b/test/pathod/test_language_base.py
index 64d4af1f..2e5d9041 100644
--- a/test/pathod/test_language_base.py
+++ b/test/pathod/test_language_base.py
@@ -67,7 +67,7 @@ class TestTokValueLiteral:
 
     def test_roundtrip(self):
         self.roundtrip("'")
-        self.roundtrip('\'')
+        self.roundtrip(r"\'")
         self.roundtrip("a")
         self.roundtrip("\"")
         # self.roundtrip("\\")
-- 
cgit v1.2.3


From d149c447fe9d3ec359271270ed1c32c2c7da6aad Mon Sep 17 00:00:00 2001
From: Maximilian Hils <git@maximilianhils.com>
Date: Wed, 25 May 2016 20:31:32 -0700
Subject: fix py3 tests

---
 netlib/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/netlib/utils.py b/netlib/utils.py
index 73b2adb3..648915fa 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -438,7 +438,7 @@ def bytes_to_escaped_str(data):
         raise ValueError("data must be bytes")
     # We always insert a double-quote here so that we get a single-quoted string back
     # https://stackoverflow.com/questions/29019340/why-does-python-use-different-quotes-for-representing-strings-depending-on-their
-    return repr('"' + data).lstrip("b")[2:-1]
+    return repr(b'"' + data).lstrip("b")[2:-1]
 
 
 def escaped_str_to_bytes(data):
-- 
cgit v1.2.3


From f3932b27dafbf040c60556de7c5739148ffd67a6 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sat, 7 May 2016 23:44:39 +0530
Subject: Py3: Import and Other misc. stuff

---
 pathod/language/__init__.py | 5 +++--
 pathod/language/base.py     | 5 ++++-
 pathod/log.py               | 4 +++-
 pathod/pathoc.py            | 6 ++----
 pathod/pathod.py            | 6 +-----
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py
index 32199e08..e27452dd 100644
--- a/pathod/language/__init__.py
+++ b/pathod/language/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+
 import itertools
 import time
 
@@ -5,8 +7,7 @@ import pyparsing as pp
 
 from . import http, http2, websockets, writer, exceptions
 
-from exceptions import *
-from base import Settings
+from .base import Settings
 assert Settings  # prevent pyflakes from messing with this
 
 
diff --git a/pathod/language/base.py b/pathod/language/base.py
index a4302998..c53e6b20 100644
--- a/pathod/language/base.py
+++ b/pathod/language/base.py
@@ -3,9 +3,12 @@ import os
 import abc
 import pyparsing as pp
 
+from six.moves import reduce
+
 from .. import utils
 from . import generators, exceptions
 
+
 class Settings(object):
 
     def __init__(
@@ -105,7 +108,7 @@ class Token(object):
 class _TokValueLiteral(Token):
 
     def __init__(self, val):
-        self.val = val.decode("string_escape")
+        self.val = val
 
     def get_generator(self, settings_):
         return self.val
diff --git a/pathod/log.py b/pathod/log.py
index f203542f..3f6aaea0 100644
--- a/pathod/log.py
+++ b/pathod/log.py
@@ -1,5 +1,7 @@
 import datetime
 
+import six
+
 import netlib.utils
 import netlib.tcp
 import netlib.http
@@ -53,7 +55,7 @@ class LogCtx(object):
                 ]
             )
         if exc_value:
-            raise exc_type, exc_value, traceback
+            six.reraise(exc_type, exc_value, traceback)
 
     def suppress(self):
         self.suppressed = True
diff --git a/pathod/pathoc.py b/pathod/pathoc.py
index a49ed351..8706868b 100644
--- a/pathod/pathoc.py
+++ b/pathod/pathoc.py
@@ -13,14 +13,12 @@ import threading
 import OpenSSL.crypto
 import six
 
-from netlib import tcp, http, certutils, websockets, socks
+from netlib import tcp, certutils, websockets, socks
 from netlib.exceptions import HttpException, TcpDisconnect, TcpTimeout, TlsException, TcpException, \
     NetlibException
 from netlib.http import http1, http2
 
-import language.http
-import language.websockets
-from . import utils, log
+from . import utils, log, language
 
 import logging
 from netlib.tutils import treq
diff --git a/pathod/pathod.py b/pathod/pathod.py
index 017ce072..af5f9e6a 100644
--- a/pathod/pathod.py
+++ b/pathod/pathod.py
@@ -6,15 +6,11 @@ import sys
 import threading
 import urllib
 
-from netlib import tcp, http, certutils, websockets
+from netlib import tcp, certutils, websockets
 from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \
     TlsException
 
 from . import version, app, language, utils, log, protocols
-import language.http
-import language.actions
-import language.exceptions
-import language.websockets
 
 
 DEFAULT_CERT_DOMAIN = "pathod.net"
-- 
cgit v1.2.3


From 88e42bab6d95eb0d1d1224c4f69caafdc03a69aa Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sat, 7 May 2016 23:45:03 +0530
Subject: Py3: inner_repr and escape_unprintables

---
 pathod/utils.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/pathod/utils.py b/pathod/utils.py
index d1e2dd00..2f9de83d 100644
--- a/pathod/utils.py
+++ b/pathod/utils.py
@@ -2,6 +2,8 @@ import os
 import sys
 import netlib.utils
 
+import six
+
 
 SIZE_UNITS = dict(
     b=1024 ** 0,
@@ -58,7 +60,7 @@ def inner_repr(s):
         Returns the inner portion of a string or unicode repr (i.e. without the
         quotes)
     """
-    if isinstance(s, unicode):
+    if six.PY2 and isinstance(s, unicode):
         return repr(s)[2:-1]
     else:
         return repr(s)[1:-1]
@@ -70,7 +72,10 @@ def escape_unprintables(s):
     """
     s = s.replace("\r\n", "PATHOD_MARKER_RN")
     s = s.replace("\n", "PATHOD_MARKER_N")
-    s = inner_repr(s)
+    if six.PY2:
+        s = inner_repr(s)
+    else:
+        s = s.encode('unicode_escape').decode('ascii')
     s = s.replace("PATHOD_MARKER_RN", "\n")
     s = s.replace("PATHOD_MARKER_N", "\n")
     return s
-- 
cgit v1.2.3


From e5cebb81fb46f21ba77cbc2a1b3a58aad5e77a66 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sun, 8 May 2016 02:55:57 +0530
Subject: Removed wrong import

---
 pathod/language/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py
index e27452dd..10da93ba 100644
--- a/pathod/language/__init__.py
+++ b/pathod/language/__init__.py
@@ -7,6 +7,7 @@ import pyparsing as pp
 
 from . import http, http2, websockets, writer, exceptions
 
+from .exceptions import *
 from .base import Settings
 assert Settings  # prevent pyflakes from messing with this
 
-- 
cgit v1.2.3


From 22e4bc1938b74b0d0bfda7cab41b54070bcd7fb9 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sun, 8 May 2016 16:20:27 +0530
Subject: Py3: Handle bytes case in inner_repr

---
 pathod/utils.py           | 3 ++-
 test/pathod/test_utils.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/pathod/utils.py b/pathod/utils.py
index 2f9de83d..b14535ca 100644
--- a/pathod/utils.py
+++ b/pathod/utils.py
@@ -60,7 +60,8 @@ def inner_repr(s):
         Returns the inner portion of a string or unicode repr (i.e. without the
         quotes)
     """
-    if six.PY2 and isinstance(s, unicode):
+    if (six.PY2 and isinstance(s, unicode)) or \
+            (six.PY3 and isinstance(s, bytes)):
         return repr(s)[2:-1]
     else:
         return repr(s)[1:-1]
diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py
index 4dcedf6e..4e891ad9 100644
--- a/test/pathod/test_utils.py
+++ b/test/pathod/test_utils.py
@@ -30,6 +30,7 @@ def test_data_path():
 def test_inner_repr():
     assert utils.inner_repr("\x66") == "\x66"
     assert utils.inner_repr(u"foo") == "foo"
+    assert utils.inner_repr(b"foo") == "foo"
 
 
 def test_escape_unprintables():
-- 
cgit v1.2.3


From 5c62fabc84d0e2e7cbb5fcde494bbfa11dd593b2 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 11 May 2016 02:07:57 +0530
Subject: Use BytesIO in pathod app

---
 pathod/app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pathod/app.py b/pathod/app.py
index aa00ed69..7e9860b9 100644
--- a/pathod/app.py
+++ b/pathod/app.py
@@ -1,6 +1,6 @@
 import logging
 import pprint
-from six.moves import cStringIO as StringIO
+import io
 import copy
 from flask import Flask, jsonify, render_template, request, abort, make_response
 from . import version, language, utils
@@ -145,7 +145,7 @@ def make_app(noapi, debug):
             args["marked"] = v.marked()
             return render(template, False, **args)
 
-        s = StringIO()
+        s = io.BytesIO()
 
         settings = copy.copy(app.config["pathod"].settings)
         settings.request_host = EXAMPLE_HOST
-- 
cgit v1.2.3


From daaa672d3974dfda9bfdad9c89b27e5e6237992c Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 11 May 2016 02:09:42 +0530
Subject: Remove Py3 specific check

---
 pathod/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/pathod/utils.py b/pathod/utils.py
index b14535ca..0cc77ab2 100644
--- a/pathod/utils.py
+++ b/pathod/utils.py
@@ -60,8 +60,7 @@ def inner_repr(s):
         Returns the inner portion of a string or unicode repr (i.e. without the
         quotes)
     """
-    if (six.PY2 and isinstance(s, unicode)) or \
-            (six.PY3 and isinstance(s, bytes)):
+    if (six.PY2 and isinstance(s, unicode)) or isinstance(s, bytes):
         return repr(s)[2:-1]
     else:
         return repr(s)[1:-1]
-- 
cgit v1.2.3


From 1699592f092b5f616ef1aa33b611ec6dabe7b255 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sun, 15 May 2016 23:27:07 +0530
Subject: Use escaped_str functions for TokValueLiteral

---
 pathod/language/base.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/pathod/language/base.py b/pathod/language/base.py
index c53e6b20..bc6da6ab 100644
--- a/pathod/language/base.py
+++ b/pathod/language/base.py
@@ -4,6 +4,7 @@ import abc
 import pyparsing as pp
 
 from six.moves import reduce
+from netlib.utils import escaped_str_to_bytes, bytes_to_escaped_str
 
 from .. import utils
 from . import generators, exceptions
@@ -108,7 +109,7 @@ class Token(object):
 class _TokValueLiteral(Token):
 
     def __init__(self, val):
-        self.val = val
+        self.val = escaped_str_to_bytes(str(val))
 
     def get_generator(self, settings_):
         return self.val
@@ -133,7 +134,7 @@ class TokValueLiteral(_TokValueLiteral):
         return v
 
     def spec(self):
-        inner = self.val.encode("string_escape")
+        inner = bytes_to_escaped_str(self.val)
         inner = inner.replace(r"\'", r"\x27")
         return "'" + inner + "'"
 
@@ -146,7 +147,7 @@ class TokValueNakedLiteral(_TokValueLiteral):
         return e.setParseAction(lambda x: cls(*x))
 
     def spec(self):
-        return self.val.encode("string_escape")
+        return bytes_to_escaped_str(self.val)
 
 
 class TokValueGenerate(Token):
@@ -164,7 +165,7 @@ class TokValueGenerate(Token):
 
     def freeze(self, settings):
         g = self.get_generator(settings)
-        return TokValueLiteral(g[:].encode("string_escape"))
+        return TokValueLiteral(bytes_to_escaped_str(g[:]))
 
     @classmethod
     def expr(cls):
@@ -224,7 +225,7 @@ class TokValueFile(Token):
         return generators.FileGenerator(s)
 
     def spec(self):
-        return "<'%s'" % self.path.encode("string_escape")
+        return "<'%s'" % bytes_to_escaped_str(self.path)
 
 
 TokValue = pp.MatchFirst(
@@ -576,4 +577,4 @@ class NestedMessage(Token):
 
     def freeze(self, settings):
         f = self.parsed.freeze(settings).spec()
-        return self.__class__(TokValueLiteral(f.encode("string_escape")))
+        return self.__class__(TokValueLiteral(bytes_to_escaped_str(f)))
-- 
cgit v1.2.3


From 1bbb178b6a0ebd534b6b9f78299118496de6b92a Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 25 May 2016 16:42:46 +0530
Subject: Remove inner_repr, fixup escape_unprintables

---
 pathod/utils.py           | 22 ++++------------------
 test/pathod/test_utils.py | 12 ++++--------
 2 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/pathod/utils.py b/pathod/utils.py
index 0cc77ab2..8c6d6290 100644
--- a/pathod/utils.py
+++ b/pathod/utils.py
@@ -2,7 +2,7 @@ import os
 import sys
 import netlib.utils
 
-import six
+from netlib.utils import bytes_to_escaped_str
 
 
 SIZE_UNITS = dict(
@@ -55,27 +55,13 @@ def xrepr(s):
     return repr(s)[1:-1]
 
 
-def inner_repr(s):
-    """
-        Returns the inner portion of a string or unicode repr (i.e. without the
-        quotes)
-    """
-    if (six.PY2 and isinstance(s, unicode)) or isinstance(s, bytes):
-        return repr(s)[2:-1]
-    else:
-        return repr(s)[1:-1]
-
-
 def escape_unprintables(s):
     """
         Like inner_repr, but preserves line breaks.
     """
-    s = s.replace("\r\n", "PATHOD_MARKER_RN")
-    s = s.replace("\n", "PATHOD_MARKER_N")
-    if six.PY2:
-        s = inner_repr(s)
-    else:
-        s = s.encode('unicode_escape').decode('ascii')
+    s = s.replace(b"\r\n", b"PATHOD_MARKER_RN")
+    s = s.replace(b"\n", b"PATHOD_MARKER_N")
+    s = bytes_to_escaped_str(s)
     s = s.replace("PATHOD_MARKER_RN", "\n")
     s = s.replace("PATHOD_MARKER_N", "\n")
     return s
diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py
index 4e891ad9..32ba3bdf 100644
--- a/test/pathod/test_utils.py
+++ b/test/pathod/test_utils.py
@@ -1,6 +1,8 @@
 from pathod import utils
 import tutils
 
+import six
+
 
 def test_membool():
     m = utils.MemBool()
@@ -27,14 +29,8 @@ def test_data_path():
     tutils.raises(ValueError, utils.data.path, "nonexistent")
 
 
-def test_inner_repr():
-    assert utils.inner_repr("\x66") == "\x66"
-    assert utils.inner_repr(u"foo") == "foo"
-    assert utils.inner_repr(b"foo") == "foo"
-
-
 def test_escape_unprintables():
     s = "".join([chr(i) for i in range(255)])
-    e = utils.escape_unprintables(s)
+    e = utils.escape_unprintables(six.b(s))
     assert e.encode('ascii')
-    assert not "PATHOD_MARKER" in e
+    assert "PATHOD_MARKER" not in e
-- 
cgit v1.2.3


From 4ec56808ddfc6e6c03ce916ca35b0d7439165650 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 25 May 2016 17:18:02 +0530
Subject: remove str()

---
 pathod/language/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pathod/language/base.py b/pathod/language/base.py
index bc6da6ab..54ca6492 100644
--- a/pathod/language/base.py
+++ b/pathod/language/base.py
@@ -109,7 +109,7 @@ class Token(object):
 class _TokValueLiteral(Token):
 
     def __init__(self, val):
-        self.val = escaped_str_to_bytes(str(val))
+        self.val = escaped_str_to_bytes(val)
 
     def get_generator(self, settings_):
         return self.val
-- 
cgit v1.2.3


From bc6cd13356a5181baafc9b01385a21991695ade3 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 26 May 2016 22:19:58 +0530
Subject: Go Python 3 by default with the bytes conversion

---
 test/pathod/test_utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py
index 32ba3bdf..8026a576 100644
--- a/test/pathod/test_utils.py
+++ b/test/pathod/test_utils.py
@@ -30,7 +30,9 @@ def test_data_path():
 
 
 def test_escape_unprintables():
-    s = "".join([chr(i) for i in range(255)])
-    e = utils.escape_unprintables(six.b(s))
+    s = bytes(range(256))
+    if six.PY2:
+        s = "".join([chr(i) for i in range(255)])
+    e = utils.escape_unprintables(s)
     assert e.encode('ascii')
     assert "PATHOD_MARKER" not in e
-- 
cgit v1.2.3


From 92317bc81d72f243095a1bfa192f568637d9bc32 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 26 May 2016 23:39:16 +0530
Subject: Enable travis Py3 testing for test_utils

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7d3fbee8..4a01174a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,9 +22,9 @@ matrix:
       git:
         depth: 9999999
     - python: 3.5
-      env: SCOPE="netlib ./test/mitmproxy/script"
+      env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py"
     - python: 3.5
-      env: SCOPE="netlib ./test/mitmproxy/script" NO_ALPN=1
+      env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" NO_ALPN=1
     - python: 2.7
       env: DOCS=1
       script: 'cd docs && make html'
-- 
cgit v1.2.3