From 069119364d6490e52ba26f2d8001c6b2bf50ab7b Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Thu, 20 Oct 2016 09:35:55 +1300
Subject: Create mitmproxy.utils hierarchy

- Add mitproxy.utils.lrucache, mitproxy.utils.data
---
 mitmproxy/addons/onboardingapp/app.py |  6 ++---
 mitmproxy/addons/termlog.py           |  4 ++--
 mitmproxy/log.py                      |  4 ++++
 mitmproxy/tools/console/common.py     |  4 ++--
 mitmproxy/tools/console/flowview.py   |  4 ++--
 mitmproxy/tools/console/master.py     |  4 ++--
 mitmproxy/utils.py                    | 40 ---------------------------------
 mitmproxy/utils/data.py               | 33 +++++++++++++++++++++++++++
 mitmproxy/utils/lrucache.py           | 32 ++++++++++++++++++++++++++
 netlib/tutils.py                      |  6 +++--
 netlib/utils.py                       | 30 -------------------------
 pathod/utils.py                       |  5 +++--
 test/mitmproxy/test_examples.py       |  5 ++---
 test/mitmproxy/test_utils.py          | 42 -----------------------------------
 test/mitmproxy/test_utils_data.py     |  7 ++++++
 test/mitmproxy/test_utils_lrucache.py | 34 ++++++++++++++++++++++++++++
 test/mitmproxy/tutils.py              |  3 ++-
 test/pathod/tutils.py                 |  5 +++--
 18 files changed, 135 insertions(+), 133 deletions(-)
 delete mode 100644 mitmproxy/utils.py
 create mode 100644 mitmproxy/utils/data.py
 create mode 100644 mitmproxy/utils/lrucache.py
 delete mode 100644 test/mitmproxy/test_utils.py
 create mode 100644 test/mitmproxy/test_utils_data.py
 create mode 100644 test/mitmproxy/test_utils_lrucache.py

diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py
index 9e07b75f..50b52214 100644
--- a/mitmproxy/addons/onboardingapp/app.py
+++ b/mitmproxy/addons/onboardingapp/app.py
@@ -4,11 +4,11 @@ import tornado.template
 import tornado.web
 import tornado.wsgi
 
-from mitmproxy import utils
+from mitmproxy.utils import data
 from mitmproxy.proxy import config
 from mitmproxy.addons import wsgiapp
 
-loader = tornado.template.Loader(utils.pkg_data.path("addons/onboardingapp/templates"))
+loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates"))
 
 
 class Adapter(tornado.wsgi.WSGIAdapter):
@@ -86,7 +86,7 @@ application = tornado.web.Application(
             r"/static/(.*)",
             tornado.web.StaticFileHandler,
             {
-                "path": utils.pkg_data.path("addons/onboardingapp/static")
+                "path": data.pkg_data.path("addons/onboardingapp/static")
             }
         ),
     ],
diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py
index 50c32044..05be32d0 100644
--- a/mitmproxy/addons/termlog.py
+++ b/mitmproxy/addons/termlog.py
@@ -1,6 +1,6 @@
 import click
 
-from mitmproxy import utils
+from mitmproxy import log
 
 
 class TermLog:
@@ -11,7 +11,7 @@ class TermLog:
         self.options = options
 
     def log(self, e):
-        if self.options.verbosity >= utils.log_tier(e.level):
+        if self.options.verbosity >= log.log_tier(e.level):
             click.secho(
                 e.msg,
                 file=self.options.tfile,
diff --git a/mitmproxy/log.py b/mitmproxy/log.py
index 8c28a9b1..c2456cf1 100644
--- a/mitmproxy/log.py
+++ b/mitmproxy/log.py
@@ -38,3 +38,7 @@ class Log:
 
     def __call__(self, text, level="info"):
         self.master.add_log(text, level)
+
+
+def log_tier(level):
+    return dict(error=0, warn=1, info=2, debug=3).get(level)
diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py
index d10d9321..dc4cfe18 100644
--- a/mitmproxy/tools/console/common.py
+++ b/mitmproxy/tools/console/common.py
@@ -7,7 +7,7 @@ import urwid
 import urwid.util
 
 import netlib
-from mitmproxy import utils
+from mitmproxy.utils import lrucache
 from mitmproxy.tools.console import signals
 from mitmproxy import export
 from netlib import human
@@ -325,7 +325,7 @@ def export_to_clip_or_file(key, scope, flow, writer):
             else:  # other keys
                 writer(exporter(flow))
 
-flowcache = utils.LRUCache(800)
+flowcache = lrucache.LRUCache(800)
 
 
 def raw_format_flow(f):
diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py
index 64caf474..afebf44e 100644
--- a/mitmproxy/tools/console/flowview.py
+++ b/mitmproxy/tools/console/flowview.py
@@ -8,7 +8,7 @@ from typing import Optional, Union  # noqa
 
 from mitmproxy import contentviews
 from mitmproxy import http
-from mitmproxy import utils
+from mitmproxy.utils import lrucache
 from mitmproxy.tools.console import common
 from mitmproxy.tools.console import flowdetailview
 from mitmproxy.tools.console import grideditor
@@ -121,7 +121,7 @@ class FlowViewHeader(urwid.WidgetWrap):
             )
 
 
-cache = utils.LRUCache(200)
+cache = lrucache.LRUCache(200)
 
 TAB_REQ = 0
 TAB_RESP = 1
diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py
index b3fd0adb..3cc721b2 100644
--- a/mitmproxy/tools/console/master.py
+++ b/mitmproxy/tools/console/master.py
@@ -21,7 +21,7 @@ from mitmproxy import exceptions
 from mitmproxy import master
 from mitmproxy import io
 from mitmproxy import flowfilter
-from mitmproxy import utils
+from mitmproxy import log
 from mitmproxy.addons import state
 import mitmproxy.options
 from mitmproxy.tools.console import flowlist
@@ -266,7 +266,7 @@ class ConsoleMaster(master.Master):
         )
 
     def sig_add_log(self, sender, e, level):
-        if self.options.verbosity < utils.log_tier(level):
+        if self.options.verbosity < log.log_tier(level):
             return
 
         if level in ("error", "warn"):
diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py
deleted file mode 100644
index fb2effd8..00000000
--- a/mitmproxy/utils.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import netlib.utils
-
-
-pkg_data = netlib.utils.Data(__name__)
-
-
-class LRUCache:
-
-    """
-        A simple LRU cache for generated values.
-    """
-
-    def __init__(self, size=100):
-        self.size = size
-        self.cache = {}
-        self.cacheList = []
-
-    def get(self, gen, *args):
-        """
-            gen: A (presumably expensive) generator function. The identity of
-            gen is NOT taken into account by the cache.
-            *args: A list of immutable arguments, used to establish identiy by
-            *the cache, and passed to gen to generate values.
-        """
-        if args in self.cache:
-            self.cacheList.remove(args)
-            self.cacheList.insert(0, args)
-            return self.cache[args]
-        else:
-            ret = gen(*args)
-            self.cacheList.insert(0, args)
-            self.cache[args] = ret
-            if len(self.cacheList) > self.size:
-                d = self.cacheList.pop()
-                self.cache.pop(d)
-            return ret
-
-
-def log_tier(level):
-    return dict(error=0, warn=1, info=2, debug=3).get(level)
diff --git a/mitmproxy/utils/data.py b/mitmproxy/utils/data.py
new file mode 100644
index 00000000..2e68d184
--- /dev/null
+++ b/mitmproxy/utils/data.py
@@ -0,0 +1,33 @@
+import os.path
+import importlib
+import inspect
+
+
+class Data:
+
+    def __init__(self, name):
+        m = importlib.import_module(name)
+        dirname = os.path.dirname(inspect.getsourcefile(m))
+        self.dirname = os.path.abspath(dirname)
+
+    def push(self, subpath):
+        """
+            Change the data object to a path relative to the module.
+        """
+        self.dirname = os.path.join(self.dirname, subpath)
+        return self
+
+    def path(self, path):
+        """
+            Returns a path to the package data housed at 'path' under this
+            module.Path can be a path to a file, or to a directory.
+
+            This function will raise ValueError if the path does not exist.
+        """
+        fullpath = os.path.join(self.dirname, path)
+        if not os.path.exists(fullpath):
+            raise ValueError("dataPath: %s does not exist." % fullpath)
+        return fullpath
+
+
+pkg_data = Data(__name__).push("..")
diff --git a/mitmproxy/utils/lrucache.py b/mitmproxy/utils/lrucache.py
new file mode 100644
index 00000000..7ad2b7f5
--- /dev/null
+++ b/mitmproxy/utils/lrucache.py
@@ -0,0 +1,32 @@
+
+
+class LRUCache:
+
+    """
+        A simple LRU cache for generated values.
+    """
+
+    def __init__(self, size=100):
+        self.size = size
+        self.cache = {}
+        self.cacheList = []
+
+    def get(self, gen, *args):
+        """
+            gen: A (presumably expensive) generator function. The identity of
+            gen is NOT taken into account by the cache.
+            *args: A list of immutable arguments, used to establish identiy by
+            *the cache, and passed to gen to generate values.
+        """
+        if args in self.cache:
+            self.cacheList.remove(args)
+            self.cacheList.insert(0, args)
+            return self.cache[args]
+        else:
+            ret = gen(*args)
+            self.cacheList.insert(0, args)
+            self.cache[args] = ret
+            if len(self.cacheList) > self.size:
+                d = self.cacheList.pop()
+                self.cache.pop(d)
+            return ret
diff --git a/netlib/tutils.py b/netlib/tutils.py
index d22fdd1c..6fa2d7b6 100644
--- a/netlib/tutils.py
+++ b/netlib/tutils.py
@@ -6,7 +6,9 @@ import shutil
 from contextlib import contextmanager
 import sys
 
-from netlib import utils, tcp, http
+from mitmproxy.utils import data
+from netlib import tcp
+from netlib import http
 
 
 def treader(bytes):
@@ -87,7 +89,7 @@ class RaisesContext:
         return True
 
 
-test_data = utils.Data(__name__)
+test_data = data.Data(__name__)
 # FIXME: Temporary workaround during repo merge.
 test_data.dirname = os.path.join(test_data.dirname, "..", "test", "netlib")
 
diff --git a/netlib/utils.py b/netlib/utils.py
index 8cd9ba6e..12b94d74 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -1,7 +1,4 @@
-import os.path
 import re
-import importlib
-import inspect
 
 
 def setbit(byte, offset, value):
@@ -48,33 +45,6 @@ class BiDi:
         return self.values.get(n, default)
 
 
-class Data:
-
-    def __init__(self, name):
-        m = importlib.import_module(name)
-        dirname = os.path.dirname(inspect.getsourcefile(m))
-        self.dirname = os.path.abspath(dirname)
-
-    def push(self, subpath):
-        """
-            Change the data object to a path relative to the module.
-        """
-        self.dirname = os.path.join(self.dirname, subpath)
-        return self
-
-    def path(self, path):
-        """
-            Returns a path to the package data housed at 'path' under this
-            module.Path can be a path to a file, or to a directory.
-
-            This function will raise ValueError if the path does not exist.
-        """
-        fullpath = os.path.join(self.dirname, path)
-        if not os.path.exists(fullpath):
-            raise ValueError("dataPath: %s does not exist." % fullpath)
-        return fullpath
-
-
 _label_valid = re.compile(b"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
 
 
diff --git a/pathod/utils.py b/pathod/utils.py
index 2d077c48..44ad1f87 100644
--- a/pathod/utils.py
+++ b/pathod/utils.py
@@ -1,6 +1,6 @@
 import os
 import sys
-import netlib.utils
+from mitmproxy.utils import data as mdata
 
 
 class MemBool:
@@ -17,7 +17,8 @@ class MemBool:
         return bool(v)
 
 
-data = netlib.utils.Data(__name__)
+# FIXME: change this name
+data = mdata.Data(__name__)
 
 
 def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):  # pragma: no cover
diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 7551c1c8..60d4a1a5 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -6,17 +6,16 @@ from mitmproxy import options
 from mitmproxy import contentviews
 from mitmproxy import proxy
 from mitmproxy.addons import script
+from mitmproxy.utils import data
 from mitmproxy import master
 
-import netlib.utils
-
 from netlib import tutils as netutils
 from netlib.http import Headers
 from netlib.http import cookies
 
 from . import tutils, mastertest
 
-example_dir = netlib.utils.Data(__name__).push("../../examples")
+example_dir = data.Data(__name__).push("../../examples")
 
 
 class ScriptError(Exception):
diff --git a/test/mitmproxy/test_utils.py b/test/mitmproxy/test_utils.py
deleted file mode 100644
index ed59f484..00000000
--- a/test/mitmproxy/test_utils.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from mitmproxy import utils
-from . import tutils
-
-utils.CERT_SLEEP_TIME = 0
-
-
-def test_pkg_data():
-    assert utils.pkg_data.path("tools/console")
-    tutils.raises("does not exist", utils.pkg_data.path, "nonexistent")
-
-
-def test_LRUCache():
-    cache = utils.LRUCache(2)
-
-    class Foo:
-        ran = False
-
-        def gen(self, x):
-            self.ran = True
-            return x
-    f = Foo()
-
-    assert not f.ran
-    assert cache.get(f.gen, 1) == 1
-    assert f.ran
-    f.ran = False
-    assert cache.get(f.gen, 1) == 1
-    assert not f.ran
-
-    f.ran = False
-    assert cache.get(f.gen, 1) == 1
-    assert not f.ran
-    assert cache.get(f.gen, 2) == 2
-    assert cache.get(f.gen, 3) == 3
-    assert f.ran
-
-    f.ran = False
-    assert cache.get(f.gen, 1) == 1
-    assert f.ran
-
-    assert len(cache.cacheList) == 2
-    assert len(cache.cache) == 2
diff --git a/test/mitmproxy/test_utils_data.py b/test/mitmproxy/test_utils_data.py
new file mode 100644
index 00000000..c6e4420e
--- /dev/null
+++ b/test/mitmproxy/test_utils_data.py
@@ -0,0 +1,7 @@
+from mitmproxy.utils import data
+from . import tutils
+
+
+def test_pkg_data():
+    assert data.pkg_data.path("tools/console")
+    tutils.raises("does not exist", data.pkg_data.path, "nonexistent")
diff --git a/test/mitmproxy/test_utils_lrucache.py b/test/mitmproxy/test_utils_lrucache.py
new file mode 100644
index 00000000..f75fb5e8
--- /dev/null
+++ b/test/mitmproxy/test_utils_lrucache.py
@@ -0,0 +1,34 @@
+from mitmproxy import utils
+
+
+def test_LRUCache():
+    cache = utils.LRUCache(2)
+
+    class Foo:
+        ran = False
+
+        def gen(self, x):
+            self.ran = True
+            return x
+    f = Foo()
+
+    assert not f.ran
+    assert cache.get(f.gen, 1) == 1
+    assert f.ran
+    f.ran = False
+    assert cache.get(f.gen, 1) == 1
+    assert not f.ran
+
+    f.ran = False
+    assert cache.get(f.gen, 1) == 1
+    assert not f.ran
+    assert cache.get(f.gen, 2) == 2
+    assert cache.get(f.gen, 3) == 3
+    assert f.ran
+
+    f.ran = False
+    assert cache.get(f.gen, 1) == 1
+    assert f.ran
+
+    assert len(cache.cacheList) == 2
+    assert len(cache.cache) == 2
diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py
index eb0c90e9..71dd20a4 100644
--- a/test/mitmproxy/tutils.py
+++ b/test/mitmproxy/tutils.py
@@ -16,6 +16,7 @@ from mitmproxy import connections
 from mitmproxy import flow
 from mitmproxy import http
 from mitmproxy import tcp
+from mitmproxy.utils import data
 
 
 def _skip_windows(*args):
@@ -208,4 +209,4 @@ def capture_stderr(command, *args, **kwargs):
     sys.stderr = out
 
 
-test_data = netlib.utils.Data(__name__)
+test_data = data.Data(__name__)
diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py
index c2243578..171d97a4 100644
--- a/test/pathod/tutils.py
+++ b/test/pathod/tutils.py
@@ -5,8 +5,9 @@ import requests
 import io
 import urllib
 
+from mitmproxy.utils import data
+
 from netlib import tcp
-from netlib import utils
 from netlib import tutils
 
 from pathod import language
@@ -142,7 +143,7 @@ tmpdir = tutils.tmpdir
 
 raises = tutils.raises
 
-test_data = utils.Data(__name__)
+test_data = data.Data(__name__)
 
 
 def render(r, settings=language.Settings()):
-- 
cgit v1.2.3