diff options
32 files changed, 253 insertions, 135 deletions
@@ -1,7 +1,7 @@ .DS_Store MANIFEST */tmp -/venv +/venv* *.py[cdo] *.swp *.swo diff --git a/.travis.yml b/.travis.yml index 7d3fbee8..4a01174a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,9 @@ matrix: git: depth: 9999999 - python: 3.5 - env: SCOPE="netlib ./test/mitmproxy/script" + env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" - python: 3.5 - env: SCOPE="netlib ./test/mitmproxy/script" NO_ALPN=1 + env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" NO_ALPN=1 - python: 2.7 env: DOCS=1 script: 'cd docs && make html' @@ -1,13 +1,16 @@ -#!/bin/bash +#!/bin/sh set -e -VENV=./venv -python -m virtualenv $VENV --always-copy -. $VENV/bin/activate -pip install -U pip setuptools -pip install -r requirements.txt +PYVERSION=$1 +VENV="venv$1" + +echo "Creating dev environment in $VENV using Python $PYVERSION" + +python$PYVERSION -m virtualenv "$VENV" --always-copy +. "$VENV/bin/activate" +pip$PYVERSION install -q -U pip setuptools +pip$PYVERSION install -q -r requirements.txt echo "" -echo "* Created virtualenv environment in $VENV." -echo "* Installed all dependencies into the virtualenv." -echo "* You can now activate the virtualenv: \`. $VENV/bin/activate\`" +echo "* Virtualenv created in $VENV and all dependencies installed." +echo "* You can now activate the $(python --version) virtualenv with this command: \`. $VENV/bin/activate\`" diff --git a/issue_template.md b/issue_template.md index 08d390e4..2392f8c6 100644 --- a/issue_template.md +++ b/issue_template.md @@ -1,3 +1,4 @@ +<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) --> ##### Steps to reproduce the problem: 1. diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py index e75aed86..1dd032be 100644 --- a/mitmproxy/console/__init__.py +++ b/mitmproxy/console/__init__.py @@ -58,7 +58,7 @@ class ConsoleState(flow.State): return f def set_limit(self, limit): - ret = flow.State.set_limit(self, limit) + ret = super(ConsoleState, self).set_limit(limit) self.set_focus(self.focus) return ret @@ -102,7 +102,7 @@ class ConsoleState(flow.State): self.focus -= 1 if self.focus < 0: self.focus = None - ret = flow.State.delete_flow(self, f) + ret = super(ConsoleState, self).delete_flow(f) self.set_focus(self.focus) return ret diff --git a/mitmproxy/console/grideditor.py b/mitmproxy/console/grideditor.py index 11ce7d02..ea26d966 100644 --- a/mitmproxy/console/grideditor.py +++ b/mitmproxy/console/grideditor.py @@ -5,7 +5,6 @@ import re import os import urwid -from netlib import odict from netlib.http import user_agents, cookies from . import common, signals diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index aae397cd..f1eabdb8 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -320,7 +320,6 @@ class DumpMaster(flow.FlowMaster): self.outfile.flush() def _process_flow(self, f): - self.state.delete_flow(f) if self.filt and not f.match(self.filt): return @@ -328,6 +327,7 @@ class DumpMaster(flow.FlowMaster): def handle_request(self, f): flow.FlowMaster.handle_request(self, f) + self.state.delete_flow(f) if f: f.reply() return f diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index a9018e16..a09a81a7 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -3,7 +3,6 @@ """ from __future__ import absolute_import -import traceback from abc import abstractmethod, ABCMeta import hashlib import sys @@ -18,12 +17,13 @@ from typing import List, Optional, Set from netlib import wsgi, odict from netlib.exceptions import HttpException from netlib.http import Headers, http1, cookies +from netlib.utils import clean_bin from . import controller, tnetstring, filt, script, version, flow_format_compat from .onboarding import app from .proxy.config import HostMatcher from .protocol.http_replay import RequestReplayThread from .exceptions import Kill, FlowReadException -from .models import ClientConnection, ServerConnection, HTTPFlow, HTTPRequest, FLOW_TYPES +from .models import ClientConnection, ServerConnection, HTTPFlow, HTTPRequest, FLOW_TYPES, TCPFlow from collections import defaultdict @@ -651,8 +651,9 @@ class FlowMaster(controller.ServerMaster): if server: self.add_server(server) self.state = state - self.server_playback = None - self.client_playback = None + self.active_flows = set() # type: Set[Flow] + self.server_playback = None # type: Optional[ServerPlaybackState] + self.client_playback = None # type: Optional[ClientPlaybackState] self.kill_nonreplay = False self.scripts = [] # type: List[script.Script] self.pause_scripts = False @@ -898,6 +899,17 @@ class FlowMaster(controller.ServerMaster): self.handle_response(f) if f.error: self.handle_error(f) + elif isinstance(f, TCPFlow): + messages = f.messages + f.messages = [] + f.reply = controller.DummyReply() + self.handle_tcp_open(f) + while messages: + f.messages.append(messages.pop(0)) + self.handle_tcp_message(f) + if f.error: + self.handle_tcp_error(f) + self.handle_tcp_close(f) else: raise NotImplementedError() @@ -1020,6 +1032,7 @@ class FlowMaster(controller.ServerMaster): return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) + self.active_flows.add(f) self.replacehooks.run(f) self.setheaders.run(f) self.process_new_request(f) @@ -1040,6 +1053,7 @@ class FlowMaster(controller.ServerMaster): return f def handle_response(self, f): + self.active_flows.discard(f) self.state.update_flow(f) self.replacehooks.run(f) self.setheaders.run(f) @@ -1085,18 +1099,47 @@ class FlowMaster(controller.ServerMaster): self.add_event('"{}" reloaded.'.format(s.filename), 'info') return ok - def handle_tcp_message(self, m): - self.run_script_hook("tcp_message", m) - m.reply() + def handle_tcp_open(self, flow): + # TODO: This would break mitmproxy currently. + # self.state.add_flow(flow) + self.active_flows.add(flow) + self.run_script_hook("tcp_open", flow) + flow.reply() + + def handle_tcp_message(self, flow): + self.run_script_hook("tcp_message", flow) + message = flow.messages[-1] + direction = "->" if message.from_client else "<-" + self.add_event("{client} {direction} tcp {direction} {server}".format( + client=repr(flow.client_conn.address), + server=repr(flow.server_conn.address), + direction=direction, + ), "info") + self.add_event(clean_bin(message.content), "debug") + flow.reply() + + def handle_tcp_error(self, flow): + self.add_event("Error in TCP connection to {}: {}".format( + repr(flow.server_conn.address), + flow.error + ), "info") + self.run_script_hook("tcp_error", flow) + flow.reply() + + def handle_tcp_close(self, flow): + self.active_flows.discard(flow) + if self.stream: + self.stream.add(flow) + self.run_script_hook("tcp_close", flow) + flow.reply() def shutdown(self): super(FlowMaster, self).shutdown() # Add all flows that are still active if self.stream: - for i in self.state.flows: - if not i.response: - self.stream.add(i) + for flow in self.active_flows: + self.stream.add(flow) self.stop_stream() self.unload_scripts() diff --git a/mitmproxy/models/__init__.py b/mitmproxy/models/__init__.py index df86eff4..3d9d9dae 100644 --- a/mitmproxy/models/__init__.py +++ b/mitmproxy/models/__init__.py @@ -7,9 +7,11 @@ from .http import ( from netlib.http import decoded from .connections import ClientConnection, ServerConnection from .flow import Flow, Error +from .tcp import TCPFlow FLOW_TYPES = dict( - http=HTTPFlow + http=HTTPFlow, + tcp=TCPFlow, ) __all__ = [ @@ -18,5 +20,6 @@ __all__ = [ "make_connect_response", "expect_continue_response", "ClientConnection", "ServerConnection", "Flow", "Error", + "TCPFlow" "FLOW_TYPES" ] diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py index 594147ec..1019c9fb 100644 --- a/mitmproxy/models/flow.py +++ b/mitmproxy/models/flow.py @@ -40,6 +40,9 @@ class Error(stateobject.StateObject): def __str__(self): return self.msg + def __repr__(self): + return self.msg + @classmethod def from_state(cls, state): # the default implementation assumes an empty constructor. Override @@ -99,6 +102,12 @@ class Flow(stateobject.StateObject): self._backup = state.pop("backup") super(Flow, self).set_state(state) + @classmethod + def from_state(cls, state): + f = cls(None, None) + f.set_state(state) + return f + def copy(self): f = copy.copy(self) diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 77a809cf..75ffbfd0 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -191,12 +191,6 @@ class HTTPFlow(Flow): response=HTTPResponse ) - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.set_state(state) - return f - def __repr__(self): s = "<HTTPFlow" for a in ("request", "response", "error", "client_conn", "server_conn"): diff --git a/mitmproxy/models/tcp.py b/mitmproxy/models/tcp.py new file mode 100644 index 00000000..7e966b95 --- /dev/null +++ b/mitmproxy/models/tcp.py @@ -0,0 +1,50 @@ +import time +from typing import List + +from netlib.utils import Serializable +from .flow import Flow + + +class TCPMessage(Serializable): + def __init__(self, from_client, content, timestamp=None): + self.content = content + self.from_client = from_client + if timestamp is None: + timestamp = time.time() + self.timestamp = timestamp + + @classmethod + def from_state(cls, state): + return cls(*state) + + def get_state(self): + return self.from_client, self.content, self.timestamp + + def set_state(self, state): + self.from_client = state.pop("from_client") + self.content = state.pop("content") + self.timestamp = state.pop("timestamp") + + def __repr__(self): + return "{direction} {content}".format( + direction="->" if self.from_client else "<-", + content=repr(self.content) + ) + + +class TCPFlow(Flow): + """ + A TCPFlow is a simplified representation of a TCP session. + """ + + def __init__(self, client_conn, server_conn, live=None): + super(TCPFlow, self).__init__("tcp", client_conn, server_conn, live) + self.messages = [] # type: List[TCPMessage] + + _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes.update( + messages=List[TCPMessage] + ) + + def __repr__(self): + return "<TCPFlow ({} messages)>".format(len(self.messages)) diff --git a/mitmproxy/protocol/rawtcp.py b/mitmproxy/protocol/rawtcp.py index 7d18025e..1b546c40 100644 --- a/mitmproxy/protocol/rawtcp.py +++ b/mitmproxy/protocol/rawtcp.py @@ -9,29 +9,26 @@ from netlib.exceptions import TcpException from netlib.tcp import ssl_read_select from netlib.utils import clean_bin from ..exceptions import ProtocolException -from .base import Layer - +from ..models import Error +from ..models.tcp import TCPFlow, TCPMessage -class TcpMessage(object): - - def __init__(self, client_conn, server_conn, sender, receiver, message): - self.client_conn = client_conn - self.server_conn = server_conn - self.sender = sender - self.receiver = receiver - self.message = message +from .base import Layer class RawTCPLayer(Layer): chunk_size = 4096 - def __init__(self, ctx, logging=True): - self.logging = logging + def __init__(self, ctx, ignore=False): + self.ignore = ignore super(RawTCPLayer, self).__init__(ctx) def __call__(self): self.connect() + if not self.ignore: + flow = TCPFlow(self.client_conn, self.server_conn, self) + self.channel.ask("tcp_open", flow) + buf = memoryview(bytearray(self.chunk_size)) client = self.client_conn.connection @@ -59,30 +56,16 @@ class RawTCPLayer(Layer): return continue - tcp_message = TcpMessage( - self.client_conn, self.server_conn, - self.client_conn if dst == server else self.server_conn, - self.server_conn if dst == server else self.client_conn, - buf[:size].tobytes()) - self.channel.ask("tcp_message", tcp_message) - dst.sendall(tcp_message.message) - - if self.logging: - # log messages are prepended with the client address, - # hence the "weird" direction string. - if dst == server: - direction = "-> tcp -> {}".format(repr(self.server_conn.address)) - else: - direction = "<- tcp <- {}".format(repr(self.server_conn.address)) - data = clean_bin(tcp_message.message) - self.log( - "{}\r\n{}".format(direction, data), - "info" - ) + tcp_message = TCPMessage(dst == server, buf[:size].tobytes()) + if not self.ignore: + flow.messages.append(tcp_message) + self.channel.ask("tcp_message", flow) + dst.sendall(tcp_message.content) except (socket.error, TcpException, SSL.Error) as e: - six.reraise( - ProtocolException, - ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e))), - sys.exc_info()[2] - ) + if not self.ignore: + flow.error = Error("TCP connection closed unexpectedly: {}".format(repr(e))) + self.channel.tell("tcp_error", flow) + finally: + if not self.ignore: + self.channel.tell("tcp_close", flow) diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index c55105ec..96e7aab6 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -65,7 +65,7 @@ class RootContext(object): else: ignore = self.config.check_ignore((client_hello.sni, 443)) if ignore: - return RawTCPLayer(top_layer, logging=False) + return RawTCPLayer(top_layer, ignore=True) # 2. Always insert a TLS layer, even if there's neither client nor server tls. # An inline script may upgrade from http to https, diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py index cda5bba6..e56ac473 100644 --- a/mitmproxy/utils.py +++ b/mitmproxy/utils.py @@ -1,11 +1,7 @@ from __future__ import (absolute_import, print_function, division) -import os import datetime -import re import time import json -import importlib -import inspect import netlib.utils diff --git a/netlib/utils.py b/netlib/utils.py index 7499f71f..648915fa 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -425,6 +425,10 @@ def safe_subn(pattern, repl, target, *args, **kwargs): def bytes_to_escaped_str(data): """ Take bytes and return a safe string that can be displayed to the user. + + Single quotes are always escaped, double quotes are never escaped: + "'" + bytes_to_escaped_str(...) + "'" + gives a valid Python string. """ # TODO: We may want to support multi-byte characters without escaping them. # One way to do would be calling .decode("utf8", "backslashreplace") first @@ -432,17 +436,23 @@ def bytes_to_escaped_str(data): if not isinstance(data, bytes): raise ValueError("data must be bytes") - return repr(data).lstrip("b")[1:-1] + # We always insert a double-quote here so that we get a single-quoted string back + # https://stackoverflow.com/questions/29019340/why-does-python-use-different-quotes-for-representing-strings-depending-on-their + return repr(b'"' + data).lstrip("b")[2:-1] def escaped_str_to_bytes(data): """ Take an escaped string and return the unescaped bytes equivalent. """ - if not isinstance(data, str): + if not isinstance(data, six.string_types): + if six.PY2: + raise ValueError("data must be str or unicode") raise ValueError("data must be str") if six.PY2: + if isinstance(data, unicode): + data = data.encode("utf8") return data.decode("string-escape") # This one is difficult - we use an undocumented Python API here diff --git a/pathod/app.py b/pathod/app.py index aa00ed69..7e9860b9 100644 --- a/pathod/app.py +++ b/pathod/app.py @@ -1,6 +1,6 @@ import logging import pprint -from six.moves import cStringIO as StringIO +import io import copy from flask import Flask, jsonify, render_template, request, abort, make_response from . import version, language, utils @@ -145,7 +145,7 @@ def make_app(noapi, debug): args["marked"] = v.marked() return render(template, False, **args) - s = StringIO() + s = io.BytesIO() settings = copy.copy(app.config["pathod"].settings) settings.request_host = EXAMPLE_HOST diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py index 32199e08..10da93ba 100644 --- a/pathod/language/__init__.py +++ b/pathod/language/__init__.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import itertools import time @@ -5,8 +7,8 @@ import pyparsing as pp from . import http, http2, websockets, writer, exceptions -from exceptions import * -from base import Settings +from .exceptions import * +from .base import Settings assert Settings # prevent pyflakes from messing with this diff --git a/pathod/language/base.py b/pathod/language/base.py index a4302998..54ca6492 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -3,9 +3,13 @@ import os import abc import pyparsing as pp +from six.moves import reduce +from netlib.utils import escaped_str_to_bytes, bytes_to_escaped_str + from .. import utils from . import generators, exceptions + class Settings(object): def __init__( @@ -105,7 +109,7 @@ class Token(object): class _TokValueLiteral(Token): def __init__(self, val): - self.val = val.decode("string_escape") + self.val = escaped_str_to_bytes(val) def get_generator(self, settings_): return self.val @@ -130,7 +134,7 @@ class TokValueLiteral(_TokValueLiteral): return v def spec(self): - inner = self.val.encode("string_escape") + inner = bytes_to_escaped_str(self.val) inner = inner.replace(r"\'", r"\x27") return "'" + inner + "'" @@ -143,7 +147,7 @@ class TokValueNakedLiteral(_TokValueLiteral): return e.setParseAction(lambda x: cls(*x)) def spec(self): - return self.val.encode("string_escape") + return bytes_to_escaped_str(self.val) class TokValueGenerate(Token): @@ -161,7 +165,7 @@ class TokValueGenerate(Token): def freeze(self, settings): g = self.get_generator(settings) - return TokValueLiteral(g[:].encode("string_escape")) + return TokValueLiteral(bytes_to_escaped_str(g[:])) @classmethod def expr(cls): @@ -221,7 +225,7 @@ class TokValueFile(Token): return generators.FileGenerator(s) def spec(self): - return "<'%s'" % self.path.encode("string_escape") + return "<'%s'" % bytes_to_escaped_str(self.path) TokValue = pp.MatchFirst( @@ -573,4 +577,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(f.encode("string_escape"))) + return self.__class__(TokValueLiteral(bytes_to_escaped_str(f))) diff --git a/pathod/log.py b/pathod/log.py index f203542f..3f6aaea0 100644 --- a/pathod/log.py +++ b/pathod/log.py @@ -1,5 +1,7 @@ import datetime +import six + import netlib.utils import netlib.tcp import netlib.http @@ -53,7 +55,7 @@ class LogCtx(object): ] ) if exc_value: - raise exc_type, exc_value, traceback + six.reraise(exc_type, exc_value, traceback) def suppress(self): self.suppressed = True diff --git a/pathod/pathoc.py b/pathod/pathoc.py index a49ed351..8706868b 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -13,14 +13,12 @@ import threading import OpenSSL.crypto import six -from netlib import tcp, http, certutils, websockets, socks +from netlib import tcp, certutils, websockets, socks from netlib.exceptions import HttpException, TcpDisconnect, TcpTimeout, TlsException, TcpException, \ NetlibException from netlib.http import http1, http2 -import language.http -import language.websockets -from . import utils, log +from . import utils, log, language import logging from netlib.tutils import treq diff --git a/pathod/pathod.py b/pathod/pathod.py index 017ce072..af5f9e6a 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -6,15 +6,11 @@ import sys import threading import urllib -from netlib import tcp, http, certutils, websockets +from netlib import tcp, certutils, websockets from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \ TlsException from . import version, app, language, utils, log, protocols -import language.http -import language.actions -import language.exceptions -import language.websockets DEFAULT_CERT_DOMAIN = "pathod.net" diff --git a/pathod/utils.py b/pathod/utils.py index d1e2dd00..8c6d6290 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -2,6 +2,8 @@ import os import sys import netlib.utils +from netlib.utils import bytes_to_escaped_str + SIZE_UNITS = dict( b=1024 ** 0, @@ -53,24 +55,13 @@ def xrepr(s): return repr(s)[1:-1] -def inner_repr(s): - """ - Returns the inner portion of a string or unicode repr (i.e. without the - quotes) - """ - if isinstance(s, unicode): - return repr(s)[2:-1] - else: - return repr(s)[1:-1] - - def escape_unprintables(s): """ Like inner_repr, but preserves line breaks. """ - s = s.replace("\r\n", "PATHOD_MARKER_RN") - s = s.replace("\n", "PATHOD_MARKER_N") - s = inner_repr(s) + s = s.replace(b"\r\n", b"PATHOD_MARKER_RN") + s = s.replace(b"\n", b"PATHOD_MARKER_N") + s = bytes_to_escaped_str(s) s = s.replace("PATHOD_MARKER_RN", "\n") s = s.replace("PATHOD_MARKER_N", "\n") return s diff --git a/test/mitmproxy/scripts/tcp_stream_modify.py b/test/mitmproxy/scripts/tcp_stream_modify.py index 93b0d5c8..d7953ef9 100644 --- a/test/mitmproxy/scripts/tcp_stream_modify.py +++ b/test/mitmproxy/scripts/tcp_stream_modify.py @@ -1,3 +1,4 @@ -def tcp_message(ctx, tm): - if tm.sender == tm.server_conn: - tm.message = tm.message.replace("foo", "bar") +def tcp_message(ctx, flow): + message = flow.messages[-1] + if not message.from_client: + message.content = message.content.replace("foo", "bar") diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index bf417423..3e78a5c4 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -680,6 +680,10 @@ class TestSerialize: for i in range(3): f = tutils.tflow(err=True) w.add(f) + f = tutils.ttcpflow() + w.add(f) + f = tutils.ttcpflow(err=True) + w.add(f) sio.seek(0) return flow.FlowReader(sio) @@ -1151,6 +1155,10 @@ class TestError: e3 = e.copy() assert e3.get_state() == e.get_state() + def test_repr(self): + e = Error("yay") + assert repr(e) + class TestClientConnection: diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 454736d4..0701d52b 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -14,7 +14,7 @@ from pathod import pathoc, pathod from mitmproxy.proxy.config import HostMatcher from mitmproxy.exceptions import Kill -from mitmproxy.models import Error, HTTPResponse +from mitmproxy.models import Error, HTTPResponse, HTTPFlow from . import tutils, tservers @@ -177,9 +177,9 @@ class TcpMixin: assert n.status_code == 304 assert i.status_code == 305 assert i2.status_code == 306 - assert any(f.response.status_code == 304 for f in self.master.state.flows) - assert not any(f.response.status_code == 305 for f in self.master.state.flows) - assert not any(f.response.status_code == 306 for f in self.master.state.flows) + assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, HTTPFlow)) + assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, HTTPFlow)) + assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, HTTPFlow)) # Test that we get the original SSL cert if self.ssl: diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 4fa519cc..c9d68cfd 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -50,9 +50,8 @@ class TestMaster(flow.FlowMaster): def clear_log(self): self.log = [] - def handle_log(self, l): - self.log.append(l.msg) - l.reply() + def add_event(self, message, level=None): + self.log.append(message) class ProxyThread(threading.Thread): diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index 2dfd710e..118f849c 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -3,6 +3,8 @@ import shutil import tempfile import argparse import sys + +from mitmproxy.models.tcp import TCPMessage from six.moves import cStringIO as StringIO from contextlib import contextmanager @@ -12,7 +14,7 @@ import netlib.utils import netlib.tutils from mitmproxy import utils, controller from mitmproxy.models import ( - ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow + ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow ) @@ -45,6 +47,26 @@ def skip_appveyor(fn): return fn +def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None): + if client_conn is True: + client_conn = tclient_conn() + if server_conn is True: + server_conn = tserver_conn() + if messages is True: + messages = [ + TCPMessage(True, b"hello"), + TCPMessage(False, b"it's me"), + ] + if err is True: + err = terr() + + f = TCPFlow(client_conn, server_conn) + f.messages = messages + f.error = err + f.reply = controller.DummyReply() + return f + + def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): """ @type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection @@ -52,7 +74,7 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): @type req: bool | None | mitmproxy.protocol.http.HTTPRequest @type resp: bool | None | mitmproxy.protocol.http.HTTPResponse @type err: bool | None | mitmproxy.protocol.primitives.Error - @return: bool | None | mitmproxy.protocol.http.HTTPFlow + @return: mitmproxy.protocol.http.HTTPFlow """ if client_conn is True: client_conn = tclient_conn() diff --git a/test/netlib/test_utils.py b/test/netlib/test_utils.py index 1d8f7b0f..fce1d0a7 100644 --- a/test/netlib/test_utils.py +++ b/test/netlib/test_utils.py @@ -178,10 +178,15 @@ def test_bytes_to_escaped_str(): assert utils.bytes_to_escaped_str(b"\b") == r"\x08" assert utils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert utils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" + assert utils.bytes_to_escaped_str(b"'") == r"\'" + assert utils.bytes_to_escaped_str(b'"') == r'"' def test_escaped_str_to_bytes(): assert utils.escaped_str_to_bytes("foo") == b"foo" - assert utils.escaped_str_to_bytes(r"\x08") == b"\b" - assert utils.escaped_str_to_bytes(r"&!?=\\)") == br"&!?=\)" - assert utils.escaped_str_to_bytes(r"ü") == b'\xc3\xbc' + assert utils.escaped_str_to_bytes("\x08") == b"\b" + assert utils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" + assert utils.escaped_str_to_bytes("ü") == b'\xc3\xbc' + assert utils.escaped_str_to_bytes(u"\\x08") == b"\b" + assert utils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)" + assert utils.escaped_str_to_bytes(u"ü") == b'\xc3\xbc'
\ No newline at end of file diff --git a/test/pathod/test_language_base.py b/test/pathod/test_language_base.py index 64d4af1f..2e5d9041 100644 --- a/test/pathod/test_language_base.py +++ b/test/pathod/test_language_base.py @@ -67,7 +67,7 @@ class TestTokValueLiteral: def test_roundtrip(self): self.roundtrip("'") - self.roundtrip('\'') + self.roundtrip(r"\'") self.roundtrip("a") self.roundtrip("\"") # self.roundtrip("\\") diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py index 4dcedf6e..8026a576 100644 --- a/test/pathod/test_utils.py +++ b/test/pathod/test_utils.py @@ -1,6 +1,8 @@ from pathod import utils import tutils +import six + def test_membool(): m = utils.MemBool() @@ -27,13 +29,10 @@ def test_data_path(): tutils.raises(ValueError, utils.data.path, "nonexistent") -def test_inner_repr(): - assert utils.inner_repr("\x66") == "\x66" - assert utils.inner_repr(u"foo") == "foo" - - def test_escape_unprintables(): - s = "".join([chr(i) for i in range(255)]) + s = bytes(range(256)) + if six.PY2: + s = "".join([chr(i) for i in range(255)]) e = utils.escape_unprintables(s) assert e.encode('ascii') - assert not "PATHOD_MARKER" in e + assert "PATHOD_MARKER" not in e @@ -3,4 +3,4 @@ Starting up - npm install - gulp -- run mitmweb and open http://localhost:8081/
\ No newline at end of file +- run mitmweb and open http://localhost:8081/ |