aboutsummaryrefslogtreecommitdiffstats
path: root/mitmproxy
diff options
context:
space:
mode:
Diffstat (limited to 'mitmproxy')
-rw-r--r--mitmproxy/contentviews.py2
-rw-r--r--mitmproxy/master.py2
-rw-r--r--mitmproxy/proxy/protocol/http2.py2
-rw-r--r--mitmproxy/proxy/protocol/http_replay.py2
-rw-r--r--mitmproxy/script/concurrent.py2
-rw-r--r--mitmproxy/stateobject.py4
-rw-r--r--mitmproxy/tcp.py4
-rw-r--r--mitmproxy/types/__init__.py0
-rw-r--r--mitmproxy/types/basethread.py14
-rw-r--r--mitmproxy/types/multidict.py298
-rw-r--r--mitmproxy/types/serializable.py32
-rw-r--r--mitmproxy/utils/__init__.py0
12 files changed, 353 insertions, 9 deletions
diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py
index 07bf09f5..a171f36b 100644
--- a/mitmproxy/contentviews.py
+++ b/mitmproxy/contentviews.py
@@ -34,7 +34,7 @@ from PIL import Image
from mitmproxy import exceptions
from mitmproxy.contrib.wbxml import ASCommandResponse
from netlib import http
-from netlib import multidict
+from mitmproxy.types import multidict
from mitmproxy.utils import strutils
from netlib.http import url
diff --git a/mitmproxy/master.py b/mitmproxy/master.py
index 1fc00112..2e57e57d 100644
--- a/mitmproxy/master.py
+++ b/mitmproxy/master.py
@@ -14,7 +14,7 @@ from mitmproxy import http
from mitmproxy import log
from mitmproxy import io
from mitmproxy.proxy.protocol import http_replay
-from netlib import basethread
+from mitmproxy.types import basethread
import netlib.http
from . import ctx as mitmproxy_ctx
diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py
index cbd8b34c..93ac51bc 100644
--- a/mitmproxy/proxy/protocol/http2.py
+++ b/mitmproxy/proxy/protocol/http2.py
@@ -15,7 +15,7 @@ from mitmproxy.proxy.protocol import base
from mitmproxy.proxy.protocol import http as httpbase
import netlib.http
from netlib import tcp
-from netlib import basethread
+from mitmproxy.types import basethread
from netlib.http import http2
diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py
index bf0697be..eef5a109 100644
--- a/mitmproxy/proxy/protocol/http_replay.py
+++ b/mitmproxy/proxy/protocol/http_replay.py
@@ -8,7 +8,7 @@ from mitmproxy import http
from mitmproxy import flow
from mitmproxy import connections
from netlib.http import http1
-from netlib import basethread
+from mitmproxy.types import basethread
# TODO: Doesn't really belong into mitmproxy.proxy.protocol...
diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py
index dc72e5b7..2fd7ad8d 100644
--- a/mitmproxy/script/concurrent.py
+++ b/mitmproxy/script/concurrent.py
@@ -4,7 +4,7 @@ offload computations from mitmproxy's main master thread.
"""
from mitmproxy import events
-from netlib import basethread
+from mitmproxy.types import basethread
class ScriptThread(basethread.BaseThread):
diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py
index f4415ecf..1ab744a5 100644
--- a/mitmproxy/stateobject.py
+++ b/mitmproxy/stateobject.py
@@ -1,7 +1,7 @@
from typing import Any
from typing import List
-import netlib.basetypes
+from mitmproxy.types import serializable
def _is_list(cls):
@@ -10,7 +10,7 @@ def _is_list(cls):
return issubclass(cls, List) or is_list_bugfix
-class StateObject(netlib.basetypes.Serializable):
+class StateObject(serializable.Serializable):
"""
An object with serializable state.
diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py
index af54c9d4..d73be98d 100644
--- a/mitmproxy/tcp.py
+++ b/mitmproxy/tcp.py
@@ -2,11 +2,11 @@ import time
from typing import List
-import netlib.basetypes
from mitmproxy import flow
+from mitmproxy.types import serializable
-class TCPMessage(netlib.basetypes.Serializable):
+class TCPMessage(serializable.Serializable):
def __init__(self, from_client, content, timestamp=None):
self.content = content
diff --git a/mitmproxy/types/__init__.py b/mitmproxy/types/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/mitmproxy/types/__init__.py
diff --git a/mitmproxy/types/basethread.py b/mitmproxy/types/basethread.py
new file mode 100644
index 00000000..a3c81d19
--- /dev/null
+++ b/mitmproxy/types/basethread.py
@@ -0,0 +1,14 @@
+import time
+import threading
+
+
+class BaseThread(threading.Thread):
+ def __init__(self, name, *args, **kwargs):
+ super().__init__(name=name, *args, **kwargs)
+ self._thread_started = time.time()
+
+ def _threadinfo(self):
+ return "%s - age: %is" % (
+ self.name,
+ int(time.time() - self._thread_started)
+ )
diff --git a/mitmproxy/types/multidict.py b/mitmproxy/types/multidict.py
new file mode 100644
index 00000000..d351e48b
--- /dev/null
+++ b/mitmproxy/types/multidict.py
@@ -0,0 +1,298 @@
+from abc import ABCMeta, abstractmethod
+
+
+try:
+ from collections.abc import MutableMapping
+except ImportError: # pragma: no cover
+ from collections import MutableMapping # Workaround for Python < 3.3
+
+from mitmproxy.types import serializable
+
+
+class _MultiDict(MutableMapping, serializable.Serializable, metaclass=ABCMeta):
+ def __repr__(self):
+ fields = (
+ repr(field)
+ for field in self.fields
+ )
+ return "{cls}[{fields}]".format(
+ cls=type(self).__name__,
+ fields=", ".join(fields)
+ )
+
+ @staticmethod
+ @abstractmethod
+ def _reduce_values(values):
+ """
+ If a user accesses multidict["foo"], this method
+ reduces all values for "foo" to a single value that is returned.
+ For example, HTTP headers are folded, whereas we will just take
+ the first cookie we found with that name.
+ """
+
+ @staticmethod
+ @abstractmethod
+ def _kconv(key):
+ """
+ This method converts a key to its canonical representation.
+ For example, HTTP headers are case-insensitive, so this method returns key.lower().
+ """
+
+ def __getitem__(self, key):
+ values = self.get_all(key)
+ if not values:
+ raise KeyError(key)
+ return self._reduce_values(values)
+
+ def __setitem__(self, key, value):
+ self.set_all(key, [value])
+
+ def __delitem__(self, key):
+ if key not in self:
+ raise KeyError(key)
+ key = self._kconv(key)
+ self.fields = tuple(
+ field for field in self.fields
+ if key != self._kconv(field[0])
+ )
+
+ def __iter__(self):
+ seen = set()
+ for key, _ in self.fields:
+ key_kconv = self._kconv(key)
+ if key_kconv not in seen:
+ seen.add(key_kconv)
+ yield key
+
+ def __len__(self):
+ return len(set(self._kconv(key) for key, _ in self.fields))
+
+ def __eq__(self, other):
+ if isinstance(other, MultiDict):
+ return self.fields == other.fields
+ return False
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def get_all(self, key):
+ """
+ Return the list of all values for a given key.
+ If that key is not in the MultiDict, the return value will be an empty list.
+ """
+ key = self._kconv(key)
+ return [
+ value
+ for k, value in self.fields
+ if self._kconv(k) == key
+ ]
+
+ def set_all(self, key, values):
+ """
+ Remove the old values for a key and add new ones.
+ """
+ key_kconv = self._kconv(key)
+
+ new_fields = []
+ for field in self.fields:
+ if self._kconv(field[0]) == key_kconv:
+ if values:
+ new_fields.append(
+ (field[0], values.pop(0))
+ )
+ else:
+ new_fields.append(field)
+ while values:
+ new_fields.append(
+ (key, values.pop(0))
+ )
+ self.fields = tuple(new_fields)
+
+ def add(self, key, value):
+ """
+ Add an additional value for the given key at the bottom.
+ """
+ self.insert(len(self.fields), key, value)
+
+ def insert(self, index, key, value):
+ """
+ Insert an additional value for the given key at the specified position.
+ """
+ item = (key, value)
+ self.fields = self.fields[:index] + (item,) + self.fields[index:]
+
+ def keys(self, multi=False):
+ """
+ Get all keys.
+
+ Args:
+ multi(bool):
+ If True, one key per value will be returned.
+ If False, duplicate keys will only be returned once.
+ """
+ return (
+ k
+ for k, _ in self.items(multi)
+ )
+
+ def values(self, multi=False):
+ """
+ Get all values.
+
+ Args:
+ multi(bool):
+ If True, all values will be returned.
+ If False, only the first value per key will be returned.
+ """
+ return (
+ v
+ for _, v in self.items(multi)
+ )
+
+ def items(self, multi=False):
+ """
+ Get all (key, value) tuples.
+
+ Args:
+ multi(bool):
+ If True, all (key, value) pairs will be returned
+ If False, only the first (key, value) pair per unique key will be returned.
+ """
+ if multi:
+ return self.fields
+ else:
+ return super().items()
+
+ def collect(self):
+ """
+ Returns a list of (key, value) tuples, where values are either
+ singular if there is only one matching item for a key, or a list
+ if there are more than one. The order of the keys matches the order
+ in the underlying fields list.
+ """
+ coll = []
+ for key in self:
+ values = self.get_all(key)
+ if len(values) == 1:
+ coll.append([key, values[0]])
+ else:
+ coll.append([key, values])
+ return coll
+
+ def to_dict(self):
+ """
+ Get the MultiDict as a plain Python dict.
+ Keys with multiple values are returned as lists.
+
+ Example:
+
+ .. code-block:: python
+
+ # Simple dict with duplicate values.
+ >>> d = MultiDict([("name", "value"), ("a", False), ("a", 42)])
+ >>> d.to_dict()
+ {
+ "name": "value",
+ "a": [False, 42]
+ }
+ """
+ return {
+ k: v for k, v in self.collect()
+ }
+
+ def get_state(self):
+ return self.fields
+
+ def set_state(self, state):
+ self.fields = tuple(tuple(x) for x in state)
+
+ @classmethod
+ def from_state(cls, state):
+ return cls(state)
+
+
+class MultiDict(_MultiDict):
+ def __init__(self, fields=()):
+ super().__init__()
+ self.fields = tuple(
+ tuple(i) for i in fields
+ )
+
+ @staticmethod
+ def _reduce_values(values):
+ return values[0]
+
+ @staticmethod
+ def _kconv(key):
+ return key
+
+
+class ImmutableMultiDict(MultiDict, metaclass=ABCMeta):
+ def _immutable(self, *_):
+ raise TypeError('{} objects are immutable'.format(self.__class__.__name__))
+
+ __delitem__ = set_all = insert = _immutable
+
+ def __hash__(self):
+ return hash(self.fields)
+
+ def with_delitem(self, key):
+ """
+ Returns:
+ An updated ImmutableMultiDict. The original object will not be modified.
+ """
+ ret = self.copy()
+ # FIXME: This is filthy...
+ super(ImmutableMultiDict, ret).__delitem__(key)
+ return ret
+
+ def with_set_all(self, key, values):
+ """
+ Returns:
+ An updated ImmutableMultiDict. The original object will not be modified.
+ """
+ ret = self.copy()
+ # FIXME: This is filthy...
+ super(ImmutableMultiDict, ret).set_all(key, values)
+ return ret
+
+ def with_insert(self, index, key, value):
+ """
+ Returns:
+ An updated ImmutableMultiDict. The original object will not be modified.
+ """
+ ret = self.copy()
+ # FIXME: This is filthy...
+ super(ImmutableMultiDict, ret).insert(index, key, value)
+ return ret
+
+
+class MultiDictView(_MultiDict):
+ """
+ The MultiDictView provides the MultiDict interface over calculated data.
+ The view itself contains no state - data is retrieved from the parent on
+ request, and stored back to the parent on change.
+ """
+ def __init__(self, getter, setter):
+ self._getter = getter
+ self._setter = setter
+ super().__init__()
+
+ @staticmethod
+ def _kconv(key):
+ # All request-attributes are case-sensitive.
+ return key
+
+ @staticmethod
+ def _reduce_values(values):
+ # We just return the first element if
+ # multiple elements exist with the same key.
+ return values[0]
+
+ @property
+ def fields(self):
+ return self._getter()
+
+ @fields.setter
+ def fields(self, value):
+ self._setter(value)
diff --git a/mitmproxy/types/serializable.py b/mitmproxy/types/serializable.py
new file mode 100644
index 00000000..49892ffc
--- /dev/null
+++ b/mitmproxy/types/serializable.py
@@ -0,0 +1,32 @@
+import abc
+
+
+class Serializable(metaclass=abc.ABCMeta):
+ """
+ Abstract Base Class that defines an API to save an object's state and restore it later on.
+ """
+
+ @classmethod
+ @abc.abstractmethod
+ def from_state(cls, state):
+ """
+ Create a new object from the given state.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def get_state(self):
+ """
+ Retrieve object state.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def set_state(self, state):
+ """
+ Set object state to the given state.
+ """
+ raise NotImplementedError()
+
+ def copy(self):
+ return self.from_state(self.get_state())
diff --git a/mitmproxy/utils/__init__.py b/mitmproxy/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/mitmproxy/utils/__init__.py
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-29 12:20:14 +0000