Move all tools into mitmproxy.tools, move models/* to top level

The primary motivation here (and for all the other moving around) is to present
a clean "front of house" to library users, and to migrate primary objects to
the top of the module hierarchy.

17 files changed, 79 insertions, 77 deletions

diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py
index ce2d2de9..e49f91bc 100644
--- a/test/mitmproxy/addons/test_dumper.py
+++ b/test/mitmproxy/addons/test_dumper.py
@@ -4,8 +4,8 @@ from .. import tutils, mastertest
 
 from mitmproxy.addons import dumper
 from mitmproxy import exceptions
-from mitmproxy import dump
-from mitmproxy import models
+from mitmproxy.tools import dump
+from mitmproxy import http
 from mitmproxy import proxy
 import netlib.tutils
 import mock
@@ -60,7 +60,7 @@ class TestDumper(mastertest.MasterTest):
         d.configure(dump.Options(tfile = sio), updated)
         flow = tutils.tflow()
         flow.request.content = None
-        flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp())
+        flow.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         flow.response.content = None
         d.response(flow)
         assert "content missing" in sio.getvalue()
diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py
index a9f18a51..1045ec29 100644
--- a/test/mitmproxy/addons/test_termlog.py
+++ b/test/mitmproxy/addons/test_termlog.py
@@ -3,7 +3,7 @@ import io
 
 from mitmproxy.addons import termlog
 from mitmproxy import log
-from mitmproxy import dump
+from mitmproxy.tools import dump
 
 
 class TestTermLog(mastertest.MasterTest):
diff --git a/test/mitmproxy/console/test_common.py b/test/mitmproxy/console/test_common.py
index ac4e0209..1a12d360 100644
--- a/test/mitmproxy/console/test_common.py
+++ b/test/mitmproxy/console/test_common.py
@@ -1,4 +1,4 @@
-import mitmproxy.console.common as common
+from mitmproxy.tools.console import common
 from .. import tutils
 
 
diff --git a/test/mitmproxy/console/test_help.py b/test/mitmproxy/console/test_help.py
index 3c1cc57c..d34f3f5d 100644
--- a/test/mitmproxy/console/test_help.py
+++ b/test/mitmproxy/console/test_help.py
@@ -1,4 +1,4 @@
-import mitmproxy.console.help as help
+import mitmproxy.tools.console.help as help
 from .. import tutils
 
 
diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py
index 46eec981..da29a2b1 100644
--- a/test/mitmproxy/console/test_master.py
+++ b/test/mitmproxy/console/test_master.py
@@ -1,9 +1,9 @@
 import gc
 
 import netlib.tutils
-from mitmproxy import console
+from mitmproxy.tools import console
 from mitmproxy import proxy
-from mitmproxy.console import common
+from mitmproxy.tools.console import common
 
 from .. import tutils, mastertest
 
diff --git a/test/mitmproxy/console/test_palettes.py b/test/mitmproxy/console/test_palettes.py
index a99730fa..7490bae0 100644
--- a/test/mitmproxy/console/test_palettes.py
+++ b/test/mitmproxy/console/test_palettes.py
@@ -1,4 +1,4 @@
-import mitmproxy.console.palettes as palettes
+import mitmproxy.tools.console.palettes as palettes
 from .. import tutils
 
 
diff --git a/test/mitmproxy/console/test_pathedit.py b/test/mitmproxy/console/test_pathedit.py
index 45c69a38..29f8de17 100644
--- a/test/mitmproxy/console/test_pathedit.py
+++ b/test/mitmproxy/console/test_pathedit.py
@@ -1,6 +1,6 @@
 import os
 from os.path import normpath
-from mitmproxy.console import pathedit
+from mitmproxy.tools.console import pathedit
 
 from mock import patch
 
diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py
index f82eec40..0b57c2d4 100644
--- a/test/mitmproxy/mastertest.py
+++ b/test/mitmproxy/mastertest.py
@@ -4,7 +4,10 @@ from . import tutils
 import netlib.tutils
 
 from mitmproxy import master
-from mitmproxy import io, proxy, models, options
+from mitmproxy import io
+from mitmproxy import proxy
+from mitmproxy import http
+from mitmproxy import options
 
 
 class TestMaster:
@@ -19,7 +22,7 @@ class MasterTest:
         master.serverconnect(f.server_conn)
         master.request(f)
         if not f.error:
-            f.response = models.HTTPResponse.wrap(
+            f.response = http.HTTPResponse.wrap(
                 netlib.tutils.tresp(content=content)
             )
             master.response(f)
diff --git a/test/mitmproxy/test_cmdline.py b/test/mitmproxy/test_cmdline.py
index 55627408..d664430a 100644
--- a/test/mitmproxy/test_cmdline.py
+++ b/test/mitmproxy/test_cmdline.py
@@ -1,5 +1,5 @@
 import argparse
-from mitmproxy import cmdline
+from mitmproxy.tools import cmdline
 from . import tutils
 
 
diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py
index c062a068..33d2423c 100644
--- a/test/mitmproxy/test_dump.py
+++ b/test/mitmproxy/test_dump.py
@@ -2,7 +2,9 @@ import os
 import io
 
 import mitmproxy.io
-from mitmproxy import dump, exceptions, proxy
+from mitmproxy.tools import dump
+from mitmproxy import exceptions
+from mitmproxy import proxy
 from . import tutils, mastertest
 
 
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index d33d395b..9ebcca34 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -8,14 +8,11 @@ from mitmproxy import flowfilter, options
 from mitmproxy.addons import state
 from mitmproxy.contrib import tnetstring
 from mitmproxy.exceptions import FlowReadException, Kill
-from mitmproxy.models import Error
-from mitmproxy.models import Flow
-from mitmproxy.models import HTTPFlow
-from mitmproxy.models import HTTPRequest
-from mitmproxy.models import HTTPResponse
+from mitmproxy import flow
+from mitmproxy import http
+from mitmproxy import connections
 from mitmproxy.proxy import ProxyConfig
 from mitmproxy.proxy.server import DummyServer
-from mitmproxy.models.connections import ClientConnection
 from mitmproxy import master
 from . import tutils
 
@@ -62,7 +59,7 @@ class TestHTTPFlow:
 
     def test_backup(self):
         f = tutils.tflow()
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         f.request.content = b"foo"
         assert not f.modified()
         f.backup()
@@ -81,20 +78,20 @@ class TestHTTPFlow:
     def test_getset_state(self):
         f = tutils.tflow(resp=True)
         state = f.get_state()
-        assert f.get_state() == HTTPFlow.from_state(
+        assert f.get_state() == http.HTTPFlow.from_state(
             state).get_state()
 
         f.response = None
-        f.error = Error("error")
+        f.error = flow.Error("error")
         state = f.get_state()
-        assert f.get_state() == HTTPFlow.from_state(
+        assert f.get_state() == http.HTTPFlow.from_state(
             state).get_state()
 
         f2 = f.copy()
         f2.id = f.id  # copy creates a different uuid
         assert f.get_state() == f2.get_state()
         assert not f == f2
-        f2.error = Error("e2")
+        f2.error = flow.Error("e2")
         assert not f == f2
         f.set_state(f2.get_state())
         assert f.get_state() == f2.get_state()
@@ -215,7 +212,7 @@ class TestState:
         assert c.add_flow(newf)
         assert c.active_flow_count() == 2
 
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         assert c.update_flow(f)
         assert c.flow_count() == 2
         assert c.active_flow_count() == 1
@@ -223,7 +220,7 @@ class TestState:
         assert not c.update_flow(None)
         assert c.active_flow_count() == 1
 
-        newf.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        newf.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         assert c.update_flow(newf)
         assert c.active_flow_count() == 0
 
@@ -231,7 +228,7 @@ class TestState:
         c = state.State()
         f = tutils.tflow()
         c.add_flow(f)
-        f.error = Error("message")
+        f.error = flow.Error("message")
         assert c.update_flow(f)
 
         c = state.State()
@@ -255,7 +252,7 @@ class TestState:
         c.set_view_filter("~s")
         assert c.filter_txt == "~s"
         assert len(c.view) == 0
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         c.update_flow(f)
         assert len(c.view) == 1
         c.set_view_filter(None)
@@ -287,7 +284,7 @@ class TestState:
     def _add_response(self, state):
         f = tutils.tflow()
         state.add_flow(f)
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         state.update_flow(f)
 
     def _add_error(self, state):
@@ -316,7 +313,7 @@ class TestState:
         c.clear()
 
         c.load_flows(flows)
-        assert isinstance(c.flows[0], Flow)
+        assert isinstance(c.flows[0], flow.Flow)
 
     def test_accept_all(self):
         c = state.State()
@@ -447,17 +444,17 @@ class TestFlowMaster:
         fm.addons.add(s)
         f = tutils.tflow(req=None)
         fm.clientconnect(f.client_conn)
-        f.request = HTTPRequest.wrap(netlib.tutils.treq())
+        f.request = http.HTTPRequest.wrap(netlib.tutils.treq())
         fm.request(f)
         assert s.flow_count() == 1
 
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
         fm.response(f)
         assert s.flow_count() == 1
 
         fm.clientdisconnect(f.client_conn)
 
-        f.error = Error("msg")
+        f.error = flow.Error("msg")
         fm.error(f)
 
         fm.shutdown()
@@ -476,7 +473,7 @@ class TestRequest:
         assert r.get_state() == r2.get_state()
 
     def test_get_url(self):
-        r = HTTPRequest.wrap(netlib.tutils.treq())
+        r = http.HTTPRequest.wrap(netlib.tutils.treq())
 
         assert r.url == "http://address:22/path"
 
@@ -497,7 +494,7 @@ class TestRequest:
         assert r.pretty_url == "https://foo.com:22/path"
 
     def test_replace(self):
-        r = HTTPRequest.wrap(netlib.tutils.treq())
+        r = http.HTTPRequest.wrap(netlib.tutils.treq())
         r.path = "path/foo"
         r.headers["Foo"] = "fOo"
         r.content = b"afoob"
@@ -507,7 +504,7 @@ class TestRequest:
         assert r.headers["boo"] == "boo"
 
     def test_constrain_encoding(self):
-        r = HTTPRequest.wrap(netlib.tutils.treq())
+        r = http.HTTPRequest.wrap(netlib.tutils.treq())
         r.headers["accept-encoding"] = "gzip, oink"
         r.constrain_encoding()
         assert "oink" not in r.headers["accept-encoding"]
@@ -517,7 +514,7 @@ class TestRequest:
         assert "oink" not in r.headers["accept-encoding"]
 
     def test_get_content_type(self):
-        resp = HTTPResponse.wrap(netlib.tutils.tresp())
+        resp = http.HTTPResponse.wrap(netlib.tutils.tresp())
         resp.headers = Headers(content_type="text/plain")
         assert resp.headers["content-type"] == "text/plain"
 
@@ -531,7 +528,7 @@ class TestResponse:
         assert resp2.get_state() == resp.get_state()
 
     def test_replace(self):
-        r = HTTPResponse.wrap(netlib.tutils.tresp())
+        r = http.HTTPResponse.wrap(netlib.tutils.tresp())
         r.headers["Foo"] = "fOo"
         r.content = b"afoob"
         assert r.replace("foo(?i)", "boo") == 3
@@ -539,7 +536,7 @@ class TestResponse:
         assert r.headers["boo"] == "boo"
 
     def test_get_content_type(self):
-        resp = HTTPResponse.wrap(netlib.tutils.tresp())
+        resp = http.HTTPResponse.wrap(netlib.tutils.tresp())
         resp.headers = Headers(content_type="text/plain")
         assert resp.headers["content-type"] == "text/plain"
 
@@ -547,13 +544,13 @@ class TestResponse:
 class TestError:
 
     def test_getset_state(self):
-        e = Error("Error")
+        e = flow.Error("Error")
         state = e.get_state()
-        assert Error.from_state(state).get_state() == e.get_state()
+        assert flow.Error.from_state(state).get_state() == e.get_state()
 
         assert e.copy()
 
-        e2 = Error("bar")
+        e2 = flow.Error("bar")
         assert not e == e2
         e.set_state(e2.get_state())
         assert e.get_state() == e2.get_state()
@@ -562,14 +559,14 @@ class TestError:
         assert e3.get_state() == e.get_state()
 
     def test_repr(self):
-        e = Error("yay")
+        e = flow.Error("yay")
         assert repr(e)
 
 
 class TestClientConnection:
     def test_state(self):
         c = tutils.tclient_conn()
-        assert ClientConnection.from_state(c.get_state()).get_state() == \
+        assert connections.ClientConnection.from_state(c.get_state()).get_state() == \
             c.get_state()
 
         c2 = tutils.tclient_conn()
diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py
index 5904ce5b..7d401184 100644
--- a/test/mitmproxy/test_proxy.py
+++ b/test/mitmproxy/test_proxy.py
@@ -2,10 +2,10 @@ import os
 import mock
 from OpenSSL import SSL
 
-from mitmproxy import cmdline
+from mitmproxy.tools import cmdline
 from mitmproxy import options
 from mitmproxy.proxy import ProxyConfig
-from mitmproxy.models.connections import ServerConnection
+from mitmproxy import connections
 from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler
 from mitmproxy.proxy import config
 from netlib.exceptions import TcpDisconnect
@@ -18,7 +18,7 @@ class TestServerConnection:
 
     def test_simple(self):
         self.d = test.Daemon()
-        sc = ServerConnection((self.d.IFACE, self.d.port))
+        sc = connections.ServerConnection((self.d.IFACE, self.d.port))
         sc.connect()
         f = tutils.tflow()
         f.server_conn = sc
@@ -36,7 +36,7 @@ class TestServerConnection:
 
     def test_terminate_error(self):
         self.d = test.Daemon()
-        sc = ServerConnection((self.d.IFACE, self.d.port))
+        sc = connections.ServerConnection((self.d.IFACE, self.d.port))
         sc.connect()
         sc.connection = mock.Mock()
         sc.connection.recv = mock.Mock(return_value=False)
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index 7616b5a7..93a82954 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -6,9 +6,10 @@ import netlib.tutils
 from mitmproxy import controller
 from mitmproxy import options
 from mitmproxy.addons import script
-from mitmproxy.models import HTTPResponse, HTTPFlow
+from mitmproxy import http
 from mitmproxy.proxy.config import HostMatcher, parse_server_spec
-from netlib import tcp, http, socks
+import netlib.http
+from netlib import tcp, socks
 from netlib.certutils import SSLCert
 from netlib.exceptions import HttpReadDisconnect, HttpException
 from netlib.http import authentication, http1
@@ -183,9 +184,9 @@ class TcpMixin:
         assert n.status_code == 304
         assert i.status_code == 305
         assert i2.status_code == 306
-        assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, HTTPFlow))
-        assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, HTTPFlow))
-        assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, HTTPFlow))
+        assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, http.HTTPFlow))
+        assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, http.HTTPFlow))
+        assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, http.HTTPFlow))
 
         # Test that we get the original SSL cert
         if self.ssl:
@@ -293,7 +294,7 @@ class TestHTTPAuth(tservers.HTTPProxyTest):
                 h'%s'='%s'
             """ % (
                 self.server.port,
-                http.authentication.BasicProxyAuth.AUTH_HEADER,
+                netlib.http.authentication.BasicProxyAuth.AUTH_HEADER,
                 authentication.assemble_http_basic_auth("basic", "test", "test")
             ))
         assert ret.status_code == 202
@@ -310,7 +311,7 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest):
                 '/p/202'
                 h'%s'='%s'
             """ % (
-                http.authentication.BasicWebsiteAuth.AUTH_HEADER,
+                netlib.http.authentication.BasicWebsiteAuth.AUTH_HEADER,
                 authentication.assemble_http_basic_auth("basic", "test", "test")
             ))
         assert ret.status_code == 202
@@ -790,7 +791,7 @@ class TestStreamRequest(tservers.HTTPProxyTest):
 class MasterFakeResponse(tservers.TestMaster):
     @controller.handler
     def request(self, f):
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = http.HTTPResponse.wrap(netlib.tutils.tresp())
 
 
 class TestFakeResponse(tservers.HTTPProxyTest):
@@ -869,7 +870,7 @@ class MasterIncomplete(tservers.TestMaster):
 
     @controller.handler
     def request(self, f):
-        resp = HTTPResponse.wrap(netlib.tutils.tresp())
+        resp = http.HTTPResponse.wrap(netlib.tutils.tresp())
         resp.content = None
         f.response = resp
 
diff --git a/test/mitmproxy/test_utils.py b/test/mitmproxy/test_utils.py
index 79b72eec..ed59f484 100644
--- a/test/mitmproxy/test_utils.py
+++ b/test/mitmproxy/test_utils.py
@@ -5,7 +5,7 @@ utils.CERT_SLEEP_TIME = 0
 
 
 def test_pkg_data():
-    assert utils.pkg_data.path("console")
+    assert utils.pkg_data.path("tools/console")
     tutils.raises("does not exist", utils.pkg_data.path, "nonexistent")
 
 
diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py
index 89354960..8fc3378a 100644
--- a/test/mitmproxy/test_web_app.py
+++ b/test/mitmproxy/test_web_app.py
@@ -1,8 +1,8 @@
 import tornado.testing
 
 from mitmproxy import proxy
-from mitmproxy.web import app
-from mitmproxy.web import master as webmaster
+from mitmproxy.tools.web import app
+from mitmproxy.tools.web import master as webmaster
 
 
 class TestApp(tornado.testing.AsyncHTTPTestCase):
diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py
index 267715ad..77280644 100644
--- a/test/mitmproxy/test_web_master.py
+++ b/test/mitmproxy/test_web_master.py
@@ -1,4 +1,4 @@
-from mitmproxy.web import master
+from mitmproxy.tools.web import master
 from mitmproxy import proxy
 from . import mastertest
 
diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py
index 2869a5f5..9724516b 100644
--- a/test/mitmproxy/tutils.py
+++ b/test/mitmproxy/tutils.py
@@ -12,11 +12,10 @@ import io
 import netlib.utils
 import netlib.tutils
 from mitmproxy import controller
-from mitmproxy.models import (
-    ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow
-)
-from mitmproxy.models.tcp import TCPMessage
-from mitmproxy.models.flow import Flow
+from mitmproxy import connections
+from mitmproxy import flow
+from mitmproxy import http
+from mitmproxy import tcp
 
 
 def _skip_windows(*args):
@@ -48,7 +47,7 @@ def skip_appveyor(fn):
         return fn
 
 
-class DummyFlow(Flow):
+class DummyFlow(flow.Flow):
     """A flow that is neither HTTP nor TCP."""
 
     def __init__(self, client_conn, server_conn, live=None):
@@ -76,13 +75,13 @@ def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None):
         server_conn = tserver_conn()
     if messages is True:
         messages = [
-            TCPMessage(True, b"hello"),
-            TCPMessage(False, b"it's me"),
+            tcp.TCPMessage(True, b"hello"),
+            tcp.TCPMessage(False, b"it's me"),
         ]
     if err is True:
         err = terr()
 
-    f = TCPFlow(client_conn, server_conn)
+    f = tcp.TCPFlow(client_conn, server_conn)
     f.messages = messages
     f.error = err
     f.reply = controller.DummyReply()
@@ -110,11 +109,11 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None):
         err = terr()
 
     if req:
-        req = HTTPRequest.wrap(req)
+        req = http.HTTPRequest.wrap(req)
     if resp:
-        resp = HTTPResponse.wrap(resp)
+        resp = http.HTTPResponse.wrap(resp)
 
-    f = HTTPFlow(client_conn, server_conn)
+    f = http.HTTPFlow(client_conn, server_conn)
     f.request = req
     f.response = resp
     f.error = err
@@ -126,7 +125,7 @@ def tclient_conn():
     """
     @return: mitmproxy.proxy.connection.ClientConnection
     """
-    c = ClientConnection.from_state(dict(
+    c = connections.ClientConnection.from_state(dict(
         address=dict(address=("address", 22), use_ipv6=True),
         clientcert=None,
         ssl_established=False,
@@ -142,7 +141,7 @@ def tserver_conn():
     """
     @return: mitmproxy.proxy.connection.ServerConnection
     """
-    c = ServerConnection.from_state(dict(
+    c = connections.ServerConnection.from_state(dict(
         address=dict(address=("address", 22), use_ipv6=True),
         source_address=dict(address=("address", 22), use_ipv6=True),
         ip_address=None,
@@ -163,7 +162,7 @@ def terr(content="error"):
     """
     @return: mitmproxy.protocol.primitives.Error
     """
-    err = Error(content)
+    err = flow.Error(content)
     return err