Utils cleanups

- Move more stuff that belongs in netlib.human
- Move some stuff to near the only use
- Zap mitmproxy.utils.timestamp(). I see the rationale, but we used it
interchangeably with time.time() throughout the project. Since time.time()
dominates in the codebase and timestamp() is such low utility, away it goes.

18 files changed, 90 insertions, 99 deletions

diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py
index acb7fc35..b450c19d 100644
--- a/mitmproxy/console/common.py
+++ b/mitmproxy/console/common.py
@@ -136,7 +136,7 @@ def raw_format_flow(f, focus, extended):
     if extended:
         req.append(
             fcol(
-                utils.format_timestamp(f["req_timestamp"]),
+                human.format_timestamp(f["req_timestamp"]),
                 "highlight"
             )
         )
diff --git a/mitmproxy/console/flowdetailview.py b/mitmproxy/console/flowdetailview.py
index e2c28e71..2a493b90 100644
--- a/mitmproxy/console/flowdetailview.py
+++ b/mitmproxy/console/flowdetailview.py
@@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division
 
 import urwid
 
-from mitmproxy import utils
 from mitmproxy.console import common, searchable
+from netlib import human
 
 
 def maybe_timestamp(base, attr):
     if base is not None and getattr(base, attr):
-        return utils.format_timestamp_with_milli(getattr(base, attr))
+        return human.format_timestamp_with_milli(getattr(base, attr))
     else:
         return "active"
 
diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py
index 42061a8c..28c57f06 100644
--- a/mitmproxy/contentviews.py
+++ b/mitmproxy/contentviews.py
@@ -28,7 +28,6 @@ from PIL import ExifTags
 from PIL import Image
 from six.moves import cStringIO as StringIO
 
-import mitmproxy.utils
 from mitmproxy import exceptions
 from mitmproxy.contrib import jsbeautifier
 from mitmproxy.contrib.wbxml import ASCommandResponse
@@ -62,6 +61,14 @@ VIEW_CUTOFF = 512
 KEY_MAX = 30
 
 
+def pretty_json(s):
+    try:
+        p = json.loads(s)
+    except ValueError:
+        return None
+    return json.dumps(p, sort_keys=True, indent=4)
+
+
 def format_dict(d):
     """
     Helper function that transforms the given dictionary into a list of
@@ -215,9 +222,9 @@ class ViewJSON(View):
     content_types = ["application/json"]
 
     def __call__(self, data, **metadata):
-        pretty_json = mitmproxy.utils.pretty_json(data)
-        if pretty_json:
-            return "JSON", format_text(pretty_json)
+        pj = pretty_json(data)
+        if pj:
+            return "JSON", format_text(pj)
 
 
 class ViewHTML(View):
diff --git a/mitmproxy/models/connections.py b/mitmproxy/models/connections.py
index 6347f488..b8e0567a 100644
--- a/mitmproxy/models/connections.py
+++ b/mitmproxy/models/connections.py
@@ -1,4 +1,5 @@
 from __future__ import absolute_import, print_function, division
+import time
 
 import copy
 import os
@@ -6,7 +7,6 @@ import os
 import six
 
 from mitmproxy import stateobject
-from mitmproxy import utils
 from netlib import certutils
 from netlib import tcp
 
@@ -39,7 +39,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
             self.clientcert = None
             self.ssl_established = None
 
-        self.timestamp_start = utils.timestamp()
+        self.timestamp_start = time.time()
         self.timestamp_end = None
         self.timestamp_ssl_setup = None
         self.protocol = None
@@ -97,11 +97,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
 
     def convert_to_ssl(self, *args, **kwargs):
         super(ClientConnection, self).convert_to_ssl(*args, **kwargs)
-        self.timestamp_ssl_setup = utils.timestamp()
+        self.timestamp_ssl_setup = time.time()
 
     def finish(self):
         super(ClientConnection, self).finish()
-        self.timestamp_end = utils.timestamp()
+        self.timestamp_end = time.time()
 
 
 class ServerConnection(tcp.TCPClient, stateobject.StateObject):
@@ -194,9 +194,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         return copy.copy(self)
 
     def connect(self):
-        self.timestamp_start = utils.timestamp()
+        self.timestamp_start = time.time()
         tcp.TCPClient.connect(self)
-        self.timestamp_tcp_setup = utils.timestamp()
+        self.timestamp_tcp_setup = time.time()
 
     def send(self, message):
         if isinstance(message, list):
@@ -218,11 +218,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
 
         self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)
         self.sni = sni
-        self.timestamp_ssl_setup = utils.timestamp()
+        self.timestamp_ssl_setup = time.time()
 
     def finish(self):
         tcp.TCPClient.finish(self)
-        self.timestamp_end = utils.timestamp()
+        self.timestamp_end = time.time()
 
 
 ServerConnection._stateobject_attributes["via"] = ServerConnection
diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py
index 7b9ec030..e2dac221 100644
--- a/mitmproxy/models/flow.py
+++ b/mitmproxy/models/flow.py
@@ -1,11 +1,11 @@
 from __future__ import absolute_import, print_function, division
 
+import time
 import copy
 import uuid
 
 from mitmproxy import exceptions
 from mitmproxy import stateobject
-from mitmproxy import utils
 from mitmproxy import version
 from mitmproxy.models.connections import ClientConnection
 from mitmproxy.models.connections import ServerConnection
@@ -34,7 +34,7 @@ class Error(stateobject.StateObject):
         @type timestamp: float
         """
         self.msg = msg
-        self.timestamp = timestamp or utils.timestamp()
+        self.timestamp = timestamp or time.time()
 
     _stateobject_attributes = dict(
         msg=str,
diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py
index ae03ab7f..187c17f6 100644
--- a/mitmproxy/protocol/http.py
+++ b/mitmproxy/protocol/http.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, print_function, division
 
+import time
 import sys
 import traceback
 
@@ -9,7 +10,6 @@ import six
 import netlib.exceptions
 from mitmproxy import exceptions
 from mitmproxy import models
-from mitmproxy import utils
 from mitmproxy.protocol import base
 from netlib import http
 from netlib import tcp
@@ -265,7 +265,7 @@ class HttpLayer(base.Layer):
             if callable(flow.response.stream):
                 chunks = flow.response.stream(chunks)
             self.send_response_body(flow.response, chunks)
-            flow.response.timestamp_end = utils.timestamp()
+            flow.response.timestamp_end = time.time()
 
     def get_response_from_server(self, flow):
         def get_response():
@@ -310,7 +310,7 @@ class HttpLayer(base.Layer):
                 flow.request,
                 flow.response
             ))
-        flow.response.timestamp_end = utils.timestamp()
+        flow.response.timestamp_end = time.time()
 
         # no further manipulation of self.server_conn beyond this point
         # we can safely set it as the final attribute value here.
diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py
index 680bc495..15785c72 100644
--- a/mitmproxy/utils.py
+++ b/mitmproxy/utils.py
@@ -1,38 +1,8 @@
 from __future__ import absolute_import, print_function, division
 
-import datetime
-import json
-import time
-
 import netlib.utils
 
 
-def timestamp():
-    """
-        Returns a serializable UTC timestamp.
-    """
-    return time.time()
-
-
-def format_timestamp(s):
-    s = time.localtime(s)
-    d = datetime.datetime.fromtimestamp(time.mktime(s))
-    return d.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def format_timestamp_with_milli(s):
-    d = datetime.datetime.fromtimestamp(s)
-    return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
-
-
-def pretty_json(s):
-    try:
-        p = json.loads(s)
-    except ValueError:
-        return None
-    return json.dumps(p, sort_keys=True, indent=4)
-
-
 pkg_data = netlib.utils.Data(__name__)
 
 
diff --git a/netlib/http/http1/assemble.py b/netlib/http/http1/assemble.py
index 00d1563b..511328f1 100644
--- a/netlib/http/http1/assemble.py
+++ b/netlib/http/http1/assemble.py
@@ -1,6 +1,6 @@
 from __future__ import absolute_import, print_function, division
 
-from netlib import utils
+import netlib.http.url
 from netlib import exceptions
 
 
@@ -82,7 +82,7 @@ def _assemble_request_headers(request_data):
     """
     headers = request_data.headers.copy()
     if "host" not in headers and request_data.scheme and request_data.host and request_data.port:
-        headers["host"] = utils.hostport(
+        headers["host"] = netlib.http.url.hostport(
             request_data.scheme,
             request_data.host,
             request_data.port
diff --git a/netlib/http/url.py b/netlib/http/url.py
index 5d461387..2fc6e7ee 100644
--- a/netlib/http/url.py
+++ b/netlib/http/url.py
@@ -78,7 +78,7 @@ def unparse(scheme, host, port, path=""):
     """
     if path == "*":
         path = ""
-    return "%s://%s%s" % (scheme, utils.hostport(scheme, host, port), path)
+    return "%s://%s%s" % (scheme, hostport(scheme, host, port), path)
 
 
 def encode(s):
@@ -94,3 +94,16 @@ def decode(s):
         Takes a urlencoded string and returns a list of (key, value) tuples.
     """
     return urllib.parse.parse_qsl(s, keep_blank_values=True)
+
+
+def hostport(scheme, host, port):
+    """
+        Returns the host component, with a port specifcation if needed.
+    """
+    if (port, scheme) in [(80, "http"), (443, "https"), (80, b"http"), (443, b"https")]:
+        return host
+    else:
+        if isinstance(host, six.binary_type):
+            return b"%s:%d" % (host, port)
+        else:
+            return "%s:%d" % (host, port)
diff --git a/netlib/human.py b/netlib/human.py
index a007adc7..72e96d30 100644
--- a/netlib/human.py
+++ b/netlib/human.py
@@ -1,3 +1,6 @@
+import datetime
+import time
+
 
 SIZE_TABLE = [
     ("b", 1024 ** 0),
@@ -48,3 +51,14 @@ def pretty_duration(secs):
             return formatter.format(secs)
     # less than 1 sec
     return "{:.0f}ms".format(secs * 1000)
+
+
+def format_timestamp(s):
+    s = time.localtime(s)
+    d = datetime.datetime.fromtimestamp(time.mktime(s))
+    return d.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def format_timestamp_with_milli(s):
+    d = datetime.datetime.fromtimestamp(s)
+    return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
diff --git a/netlib/utils.py b/netlib/utils.py
index b4b99679..79340cbd 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -4,8 +4,6 @@ import re
 import importlib
 import inspect
 
-import six
-
 
 def setbit(byte, offset, value):
     """
@@ -94,16 +92,3 @@ def is_valid_host(host):
 
 def is_valid_port(port):
     return 0 <= port <= 65535
-
-
-def hostport(scheme, host, port):
-    """
-        Returns the host component, with a port specifcation if needed.
-    """
-    if (port, scheme) in [(80, "http"), (443, "https"), (80, b"http"), (443, b"https")]:
-        return host
-    else:
-        if isinstance(host, six.binary_type):
-            return b"%s:%d" % (host, port)
-        else:
-            return "%s:%d" % (host, port)
diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py
index 5ca2ca85..5bb0cdd3 100644
--- a/pathod/pathod_cmdline.py
+++ b/pathod/pathod_cmdline.py
@@ -5,7 +5,16 @@ import os.path
 import re
 
 from netlib import tcp, human
-from . import pathod, version, utils
+from . import pathod, version
+
+
+def parse_anchor_spec(s):
+    """
+        Return a tuple, or None on error.
+    """
+    if "=" not in s:
+        return None
+    return tuple(s.split("=", 1))
 
 
 def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
@@ -188,7 +197,7 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
 
     alst = []
     for i in args.anchors:
-        parts = utils.parse_anchor_spec(i)
+        parts = parse_anchor_spec(i)
         if not parts:
             return parser.error("Invalid anchor specification: %s" % i)
         alst.append(parts)
diff --git a/pathod/utils.py b/pathod/utils.py
index 3276198a..9b220e9a 100644
--- a/pathod/utils.py
+++ b/pathod/utils.py
@@ -17,15 +17,6 @@ class MemBool(object):
         return bool(v)
 
 
-def parse_anchor_spec(s):
-    """
-        Return a tuple, or None on error.
-    """
-    if "=" not in s:
-        return None
-    return tuple(s.split("=", 1))
-
-
 data = netlib.utils.Data(__name__)
 
 
diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py
index 9142bdad..48d6c307 100644
--- a/test/mitmproxy/test_contentview.py
+++ b/test/mitmproxy/test_contentview.py
@@ -1,3 +1,5 @@
+import json
+
 from mitmproxy.exceptions import ContentViewException
 from netlib.http import Headers
 from netlib.odict import ODict
@@ -274,3 +276,9 @@ if cv.ViewProtobuf.is_available():
 
 def test_get_by_shortcut():
     assert cv.get_by_shortcut("h")
+
+
+def test_pretty_json():
+    s = json.dumps({"foo": 1})
+    assert cv.pretty_json(s)
+    assert not cv.pretty_json("moo")
diff --git a/test/mitmproxy/test_utils.py b/test/mitmproxy/test_utils.py
index c01b5f2a..79b72eec 100644
--- a/test/mitmproxy/test_utils.py
+++ b/test/mitmproxy/test_utils.py
@@ -1,29 +1,14 @@
-import json
 from mitmproxy import utils
 from . import tutils
 
 utils.CERT_SLEEP_TIME = 0
 
 
-def test_format_timestamp():
-    assert utils.format_timestamp(utils.timestamp())
-
-
-def test_format_timestamp_with_milli():
-    assert utils.format_timestamp_with_milli(utils.timestamp())
-
-
 def test_pkg_data():
     assert utils.pkg_data.path("console")
     tutils.raises("does not exist", utils.pkg_data.path, "nonexistent")
 
 
-def test_pretty_json():
-    s = json.dumps({"foo": 1})
-    assert utils.pretty_json(s)
-    assert not utils.pretty_json("moo")
-
-
 def test_LRUCache():
     cache = utils.LRUCache(2)
 
diff --git a/test/netlib/test_human.py b/test/netlib/test_human.py
index 2a5c2a85..bb97dc54 100644
--- a/test/netlib/test_human.py
+++ b/test/netlib/test_human.py
@@ -1,6 +1,15 @@
+import time
 from netlib import human, tutils
 
 
+def test_format_timestamp():
+    assert human.format_timestamp(time.time())
+
+
+def test_format_timestamp_with_milli():
+    assert human.format_timestamp_with_milli(time.time())
+
+
 def test_parse_size():
     assert human.parse_size("0") == 0
     assert human.parse_size("0b") == 0
diff --git a/test/pathod/test_pathod_cmdline.py b/test/pathod/test_pathod_cmdline.py
index 3c0918ef..18d54c82 100644
--- a/test/pathod/test_pathod_cmdline.py
+++ b/test/pathod/test_pathod_cmdline.py
@@ -3,6 +3,11 @@ import tutils
 import mock
 
 
+def test_parse_anchor_spec():
+    assert cmdline.parse_anchor_spec("foo=200") == ("foo", "200")
+    assert cmdline.parse_anchor_spec("foo") is None
+
+
 @mock.patch("argparse.ArgumentParser.error")
 def test_pathod(perror):
     assert cmdline.args_pathod(["pathod"])
diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py
index a46a523a..2bb82fe7 100644
--- a/test/pathod/test_utils.py
+++ b/test/pathod/test_utils.py
@@ -11,10 +11,5 @@ def test_membool():
     assert m.v == 2
 
 
-def test_parse_anchor_spec():
-    assert utils.parse_anchor_spec("foo=200") == ("foo", "200")
-    assert utils.parse_anchor_spec("foo") is None
-
-
 def test_data_path():
     tutils.raises(ValueError, utils.data.path, "nonexistent")