diff options
66 files changed, 601 insertions, 488 deletions
diff --git a/mitmproxy/cmdline.py b/mitmproxy/cmdline.py index 5ea76e44..a873143d 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/cmdline.py @@ -1,15 +1,17 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division + +import base64 import os import re -import base64 import configargparse -from netlib.tcp import Address, sslversion_choices -import netlib.http.url +from mitmproxy import filt +from mitmproxy import version +from mitmproxy.proxy import config from netlib import human -from . import filt, version -from .proxy import config +from netlib import tcp +from netlib.http import url APP_HOST = "mitm.it" APP_PORT = 80 @@ -104,17 +106,17 @@ def parse_setheader(s): return _parse_hook(s) -def parse_server_spec(url): +def parse_server_spec(spec): try: - p = netlib.http.url.parse(url) + p = url.parse(spec) if p[0] not in ("http", "https"): raise ValueError() except ValueError: raise configargparse.ArgumentTypeError( - "Invalid server specification: %s" % url + "Invalid server specification: %s" % spec ) - address = Address(p[1:3]) + address = tcp.Address(p[1:3]) scheme = p[0].lower() return config.ServerSpec(scheme, address) @@ -477,14 +479,14 @@ def proxy_ssl_options(parser): group.add_argument( "--ssl-version-client", dest="ssl_version_client", default="secure", action="store", - choices=sslversion_choices.keys(), + choices=tcp.sslversion_choices.keys(), help="Set supported SSL/TLS versions for client connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( "--ssl-version-server", dest="ssl_version_server", default="secure", action="store", - choices=sslversion_choices.keys(), + choices=tcp.sslversion_choices.keys(), help="Set supported SSL/TLS versions for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py index 1a70157a..00fb4b1b 100644 --- a/mitmproxy/console/__init__.py +++ b/mitmproxy/console/__init__.py @@ -1,8 +1,7 @@ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, division import mailcap import mimetypes -import tempfile import os import os.path import shlex @@ -10,16 +9,28 @@ import signal import stat import subprocess import sys +import tempfile import traceback -import urwid import weakref -from netlib import tcp +import urwid -from .. import flow, script, contentviews, controller -from . import flowlist, flowview, help, window, signals, options -from . import grideditor, palettes, statusbar, palettepicker -from ..exceptions import FlowReadException, ScriptException +from mitmproxy import contentviews +from mitmproxy import controller +from mitmproxy import exceptions +from mitmproxy import flow +from mitmproxy import script +from mitmproxy.console import flowlist +from mitmproxy.console import flowview +from mitmproxy.console import grideditor +from mitmproxy.console import help +from mitmproxy.console import options +from mitmproxy.console import palettepicker +from mitmproxy.console import palettes +from mitmproxy.console import signals +from mitmproxy.console import statusbar +from mitmproxy.console import window +from netlib import tcp EVENTLOG_SIZE = 500 @@ -231,7 +242,7 @@ class ConsoleMaster(flow.FlowMaster): for i in options.scripts: try: self.load_script(i) - except ScriptException as e: + except exceptions.ScriptException as e: print("Script load error: {}".format(e), file=sys.stderr) sys.exit(1) @@ -352,7 +363,7 @@ class ConsoleMaster(flow.FlowMaster): """ try: return flow.read_flows_from_paths(path) - except FlowReadException as e: + except exceptions.FlowReadException as e: signals.status_message.send(message=e.strerror) def client_playback_path(self, path): @@ -636,7 +647,7 @@ class ConsoleMaster(flow.FlowMaster): reterr = None try: flow.FlowMaster.load_flows_file(self, path) - except FlowReadException as e: + except exceptions.FlowReadException as e: reterr = str(e) signals.flowlist_change.send(self) return reterr @@ -666,7 +677,7 @@ class ConsoleMaster(flow.FlowMaster): for command in commands: try: self.load_script(command) - except ScriptException as e: + except exceptions.ScriptException as e: signals.status_message.send( message='Error loading "{}".'.format(command) ) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index a3ed5dc8..acb7fc35 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -1,18 +1,17 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division + +import os import urwid import urwid.util -import os import netlib +from mitmproxy import flow +from mitmproxy import models +from mitmproxy import utils +from mitmproxy.console import signals from netlib import human -from .. import utils -from .. import flow -from ..models import decoded -from . import signals - - try: import pyperclip except: @@ -260,7 +259,7 @@ def copy_flow_format_data(part, scope, flow): if scope in ("q", "a"): if flow.request.content is None: return None, "Request content is missing" - with decoded(flow.request): + with models.decoded(flow.request): if part == "h": data += netlib.http.http1.assemble_request(flow.request) elif part == "c": @@ -273,7 +272,7 @@ def copy_flow_format_data(part, scope, flow): if scope in ("s", "a") and flow.response: if flow.response.content is None: return None, "Response content is missing" - with decoded(flow.response): + with models.decoded(flow.response): if part == "h": data += netlib.http.http1.assemble_response(flow.response) elif part == "c": diff --git a/mitmproxy/console/flowdetailview.py b/mitmproxy/console/flowdetailview.py index ca083b10..e2c28e71 100644 --- a/mitmproxy/console/flowdetailview.py +++ b/mitmproxy/console/flowdetailview.py @@ -1,7 +1,9 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division + import urwid -from . import common, searchable -from .. import utils + +from mitmproxy import utils +from mitmproxy.console import common, searchable def maybe_timestamp(base, attr): diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index 1e65e3eb..eb1e76fb 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -1,9 +1,10 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division + import urwid import netlib.http.url - -from . import common, signals +from mitmproxy.console import common +from mitmproxy.console import signals def _mkhelp(): diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index 2010cecd..e9b23176 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -1,17 +1,25 @@ -from __future__ import absolute_import, division +from __future__ import absolute_import, print_function, division + +import math import os -import traceback import sys +import traceback -import math import urwid -from netlib.http import Headers, status_codes -from . import common, grideditor, signals, searchable, tabs -from . import flowdetailview -from .. import utils, controller, contentviews -from ..models import HTTPRequest, HTTPResponse, decoded -from ..exceptions import ContentViewException +from mitmproxy import contentviews +from mitmproxy import controller +from mitmproxy import exceptions +from mitmproxy import models +from mitmproxy import utils +from mitmproxy.console import common +from mitmproxy.console import flowdetailview +from mitmproxy.console import grideditor +from mitmproxy.console import searchable +from mitmproxy.console import signals +from mitmproxy.console import tabs +from netlib.http import Headers +from netlib.http import status_codes class SearchError(Exception): @@ -193,12 +201,12 @@ class FlowView(tabs.Tabs): try: query = None - if isinstance(message, HTTPRequest): + if isinstance(message, models.HTTPRequest): query = message.query description, lines = contentviews.get_content_view( viewmode, message.content, headers=message.headers, query=query ) - except ContentViewException: + except exceptions.ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() signals.add_event(s, "error") description, lines = contentviews.get_content_view( @@ -207,7 +215,7 @@ class FlowView(tabs.Tabs): description = description.replace("Raw", "Couldn't parse: falling back to Raw") # Give hint that you have to tab for the response. - if description == "No content" and isinstance(message, HTTPRequest): + if description == "No content" and isinstance(message, models.HTTPRequest): description = "No request content (press tab to view response)" # If the users has a wide terminal, he gets fewer lines; this should not be an issue. @@ -372,7 +380,7 @@ class FlowView(tabs.Tabs): message = self.flow.request else: if not self.flow.response: - self.flow.response = HTTPResponse( + self.flow.response = models.HTTPResponse( self.flow.request.http_version, 200, "OK", Headers(), "" ) @@ -399,7 +407,7 @@ class FlowView(tabs.Tabs): ) ) if part == "r": - with decoded(message): + with models.decoded(message): # Fix an issue caused by some editors when editing a # request/response body. Many editors make it hard to save a # file without a terminating newline on the last line. When diff --git a/mitmproxy/console/grideditor.py b/mitmproxy/console/grideditor.py index ea26d966..9fa51ccb 100644 --- a/mitmproxy/console/grideditor.py +++ b/mitmproxy/console/grideditor.py @@ -1,15 +1,18 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division import copy -import re import os -import urwid - -from netlib.http import user_agents, cookies +import re -from . import common, signals -from .. import utils, filt, script +import urwid +from mitmproxy import filt +from mitmproxy import script +from mitmproxy import utils +from mitmproxy.console import common +from mitmproxy.console import signals +from netlib.http import cookies +from netlib.http import user_agents FOOTER = [ ('heading_key', "enter"), ":edit ", diff --git a/mitmproxy/console/help.py b/mitmproxy/console/help.py index 0c264ebf..26cb4ed3 100644 --- a/mitmproxy/console/help.py +++ b/mitmproxy/console/help.py @@ -1,9 +1,11 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division import urwid -from . import common, signals -from .. import filt, version +from mitmproxy import filt +from mitmproxy import version +from mitmproxy.console import common +from mitmproxy.console import signals footer = [ ("heading", 'mitmproxy v%s ' % version.VERSION), diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index 5c9e0cc9..5a01c9d5 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -1,8 +1,13 @@ +from __future__ import absolute_import, print_function, division + import urwid -from .. import contentviews -from . import common, signals, grideditor -from . import select, palettes +from mitmproxy import contentviews +from mitmproxy.console import common +from mitmproxy.console import grideditor +from mitmproxy.console import palettes +from mitmproxy.console import select +from mitmproxy.console import signals footer = [ ('heading_key', "enter/space"), ":toggle ", diff --git a/mitmproxy/console/palettepicker.py b/mitmproxy/console/palettepicker.py index 51ad0606..f2acba0a 100644 --- a/mitmproxy/console/palettepicker.py +++ b/mitmproxy/console/palettepicker.py @@ -1,6 +1,11 @@ +from __future__ import absolute_import, print_function, division + import urwid -from . import select, common, palettes, signals +from mitmproxy.console import common +from mitmproxy.console import palettes +from mitmproxy.console import select +from mitmproxy.console import signals footer = [ ('heading_key', "enter/space"), ":select", diff --git a/mitmproxy/console/palettes.py b/mitmproxy/console/palettes.py index bd370181..36cc3ac0 100644 --- a/mitmproxy/console/palettes.py +++ b/mitmproxy/console/palettes.py @@ -3,6 +3,7 @@ # # http://urwid.org/manual/displayattributes.html # +from __future__ import absolute_import, print_function, division class Palette: diff --git a/mitmproxy/console/pathedit.py b/mitmproxy/console/pathedit.py index 4447070b..0eae9123 100644 --- a/mitmproxy/console/pathedit.py +++ b/mitmproxy/console/pathedit.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function, division + import glob import os.path diff --git a/mitmproxy/console/searchable.py b/mitmproxy/console/searchable.py index cff1f0a1..c60d1cd9 100644 --- a/mitmproxy/console/searchable.py +++ b/mitmproxy/console/searchable.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import, print_function, division + import urwid -from . import signals +from mitmproxy.console import signals class Highlight(urwid.AttrMap): diff --git a/mitmproxy/console/select.py b/mitmproxy/console/select.py index 7c6d1e96..091f07a2 100644 --- a/mitmproxy/console/select.py +++ b/mitmproxy/console/select.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import, print_function, division + import urwid -from . import common +from mitmproxy.console import common class _OptionWidget(urwid.WidgetWrap): diff --git a/mitmproxy/console/signals.py b/mitmproxy/console/signals.py index 6a439bf3..b57ebf0c 100644 --- a/mitmproxy/console/signals.py +++ b/mitmproxy/console/signals.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function, division + import blinker # Show a status message in the action bar diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index d57ac69c..b3e1517f 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -1,10 +1,14 @@ +from __future__ import absolute_import, print_function, division + import os.path import urwid import netlib.utils +from mitmproxy.console import common +from mitmproxy.console import pathedit +from mitmproxy.console import signals from netlib import human -from . import pathedit, signals, common class ActionBar(urwid.WidgetWrap): diff --git a/mitmproxy/console/tabs.py b/mitmproxy/console/tabs.py index b5423038..bfcdeba3 100644 --- a/mitmproxy/console/tabs.py +++ b/mitmproxy/console/tabs.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function, division + import urwid diff --git a/mitmproxy/console/window.py b/mitmproxy/console/window.py index 47c284e4..25780daf 100644 --- a/mitmproxy/console/window.py +++ b/mitmproxy/console/window.py @@ -1,5 +1,8 @@ +from __future__ import absolute_import, print_function, division + import urwid -from . import signals + +from mitmproxy.console import signals class Window(urwid.Frame): diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 373c9514..6215fa4d 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -12,27 +12,32 @@ use. For HTTP, the message headers are passed as the ``headers`` keyword argumen requests, the query parameters are passed as the ``query`` keyword argument. """ -from __future__ import (absolute_import, print_function, division) -from six.moves import cStringIO as StringIO +from __future__ import absolute_import, print_function, division + +import datetime import json import logging import subprocess import sys -import lxml.html -import lxml.etree -import datetime -from PIL import Image -from PIL.ExifTags import TAGS + +import PIL import html2text +import lxml.etree +import lxml.html import six -from netlib.odict import ODict -from netlib import encoding, http +from PIL import ExifTags +from six.moves import cStringIO as StringIO + +from mitmproxy import exceptions +from mitmproxy import utils +from mitmproxy.contrib import jsbeautifier +from mitmproxy.contrib.wbxml import ASCommandResponse +from netlib import encoding +from netlib import http +from netlib import odict from netlib.http import url -from netlib.utils import clean_bin, hexdump -from . import utils -from .exceptions import ContentViewException -from .contrib import jsbeautifier -from .contrib.wbxml.ASCommandResponse import ASCommandResponse +from netlib.utils import clean_bin +from netlib.utils import hexdump try: import pyamf @@ -259,7 +264,7 @@ class ViewURLEncoded(View): def __call__(self, data, **metadata): d = url.decode(data) - return "URLEncoded form", format_dict(ODict(d)) + return "URLEncoded form", format_dict(odict.ODict(d)) class ViewMultipart(View): @@ -270,7 +275,7 @@ class ViewMultipart(View): @staticmethod def _format(v): yield [("highlight", "Form data:\n")] - for message in format_dict(ODict(v)): + for message in format_dict(odict.ODict(v)): yield message def __call__(self, data, **metadata): @@ -398,7 +403,7 @@ class ViewImage(View): def __call__(self, data, **metadata): try: - img = Image.open(StringIO(data)) + img = PIL.Image.open(StringIO(data)) except IOError: return None parts = [ @@ -415,11 +420,11 @@ class ViewImage(View): ex = img._getexif() if ex: for i in sorted(ex.keys()): - tag = TAGS.get(i, i) + tag = ExifTags.TAGS.get(i, i) parts.append( (str(tag), str(ex[i])) ) - fmt = format_dict(ODict(parts)) + fmt = format_dict(odict.ODict(parts)) return "%s image" % img.format, fmt @@ -490,7 +495,7 @@ class ViewWBXML(View): def __call__(self, data, **metadata): try: - parser = ASCommandResponse(data) + parser = ASCommandResponse.ASCommandResponse(data) parsedContent = parser.xmlString if parsedContent: return "WBXML", format_text(parsedContent) @@ -519,12 +524,12 @@ def add(view): # TODO: auto-select a different name (append an integer?) for i in views: if i.name == view.name: - raise ContentViewException("Duplicate view: " + view.name) + raise exceptions.ContentViewException("Duplicate view: " + view.name) # TODO: the UI should auto-prompt for a replacement shortcut for prompt in view_prompts: if prompt[1] == view.prompt[1]: - raise ContentViewException("Duplicate view shortcut: " + view.prompt[1]) + raise exceptions.ContentViewException("Duplicate view shortcut: " + view.prompt[1]) views.append(view) @@ -611,8 +616,8 @@ def get_content_view(viewmode, data, **metadata): # Third-party viewers can fail in unexpected ways... except Exception as e: six.reraise( - ContentViewException, - ContentViewException(str(e)), + exceptions.ContentViewException, + exceptions.ContentViewException(str(e)), sys.exc_info()[2] ) if not ret: diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index a9e101eb..1498c3ad 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -1,9 +1,11 @@ -from __future__ import absolute_import -from six.moves import queue -import threading +from __future__ import absolute_import, print_function, division + import functools +import threading + +from six.moves import queue -from . import exceptions +from mitmproxy import exceptions Events = frozenset([ "clientconnect", diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 0f54b4d4..f5d6725c 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -1,13 +1,19 @@ -from __future__ import absolute_import, print_function -import traceback +from __future__ import absolute_import, print_function, division + +import itertools import sys +import traceback + import click -import itertools -from netlib import tcp, human -from netlib.utils import bytes_to_escaped_str -from . import flow, filt, contentviews, controller -from .exceptions import ContentViewException, FlowReadException, ScriptException +from mitmproxy import contentviews +from mitmproxy import controller +from mitmproxy import exceptions +from mitmproxy import filt +from mitmproxy import flow +from netlib import human +from netlib import tcp +from netlib import utils class DumpError(Exception): @@ -127,13 +133,13 @@ class DumpMaster(flow.FlowMaster): for command in scripts: try: self.load_script(command, use_reloader=True) - except ScriptException as e: + except exceptions.ScriptException as e: raise DumpError(str(e)) if options.rfile: try: self.load_flows_file(options.rfile) - except FlowReadException as v: + except exceptions.FlowReadException as v: self.add_event("Flow file corrupted.", "error") raise DumpError(v) @@ -147,7 +153,7 @@ class DumpMaster(flow.FlowMaster): """ try: return flow.read_flows_from_paths(paths) - except FlowReadException as e: + except exceptions.FlowReadException as e: raise DumpError(str(e)) def add_event(self, e, level="info"): @@ -175,8 +181,8 @@ class DumpMaster(flow.FlowMaster): if self.o.flow_detail >= 2: headers = "\r\n".join( "{}: {}".format( - click.style(bytes_to_escaped_str(k), fg="blue", bold=True), - click.style(bytes_to_escaped_str(v), fg="blue")) + click.style(utils.bytes_to_escaped_str(k), fg="blue", bold=True), + click.style(utils.bytes_to_escaped_str(v), fg="blue")) for k, v in message.headers.fields ) self.echo(headers, indent=4) @@ -192,7 +198,7 @@ class DumpMaster(flow.FlowMaster): message.content, headers=message.headers ) - except ContentViewException: + except exceptions.ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() self.add_event(s, "debug") type, lines = contentviews.get_content_view( @@ -238,7 +244,7 @@ class DumpMaster(flow.FlowMaster): stickycookie = "" if flow.client_conn: - client = click.style(bytes_to_escaped_str(flow.client_conn.address.host), bold=True) + client = click.style(utils.bytes_to_escaped_str(flow.client_conn.address.host), bold=True) else: client = click.style("[replay]", fg="yellow", bold=True) @@ -247,12 +253,12 @@ class DumpMaster(flow.FlowMaster): GET="green", DELETE="red" ).get(method.upper(), "magenta") - method = click.style(bytes_to_escaped_str(method), fg=method_color, bold=True) + method = click.style(utils.bytes_to_escaped_str(method), fg=method_color, bold=True) if self.showhost: url = flow.request.pretty_url else: url = flow.request.url - url = click.style(bytes_to_escaped_str(url), bold=True) + url = click.style(utils.bytes_to_escaped_str(url), bold=True) httpversion = "" if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): @@ -282,7 +288,7 @@ class DumpMaster(flow.FlowMaster): elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - reason = click.style(bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True) + reason = click.style(utils.bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True) if flow.response.content is None: size = "(content missing)" diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index b1d2c03a..d97b9498 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -5,11 +5,10 @@ Every Exception mitmproxy raises shall be a subclass of ProxyException. See also: http://lucumr.pocoo.org/2014/10/16/on-error-handling/ """ -from __future__ import (absolute_import, print_function, division) - -import traceback +from __future__ import absolute_import, print_function, division import sys +import traceback class ProxyException(Exception): diff --git a/mitmproxy/filt.py b/mitmproxy/filt.py index 6228d364..d98e3749 100644 --- a/mitmproxy/filt.py +++ b/mitmproxy/filt.py @@ -31,9 +31,11 @@ ~c CODE Response code. rex Equivalent to ~u rex """ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, division + import re import sys + import pyparsing as pp diff --git a/mitmproxy/flow/export.py b/mitmproxy/flow/export.py index d71ac609..97cee984 100644 --- a/mitmproxy/flow/export.py +++ b/mitmproxy/flow/export.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function, division + import json import re from textwrap import dedent diff --git a/mitmproxy/flow/io.py b/mitmproxy/flow/io.py index f31c8e5b..cd3d9986 100644 --- a/mitmproxy/flow/io.py +++ b/mitmproxy/flow/io.py @@ -1,7 +1,10 @@ +from __future__ import absolute_import, print_function, division + import os -from mitmproxy import tnetstring, models -from mitmproxy.exceptions import FlowReadException +from mitmproxy import exceptions +from mitmproxy import models +from mitmproxy import tnetstring from mitmproxy.flow import io_compat @@ -38,17 +41,17 @@ class FlowReader: try: data = io_compat.migrate_flow(data) except ValueError as e: - raise FlowReadException(str(e)) + raise exceptions.FlowReadException(str(e)) if can_tell: off = self.fo.tell() if data["type"] not in models.FLOW_TYPES: - raise FlowReadException("Unknown flow type: {}".format(data["type"])) + raise exceptions.FlowReadException("Unknown flow type: {}".format(data["type"])) yield models.FLOW_TYPES[data["type"]].from_state(data) except ValueError: # Error is due to EOF if can_tell and self.fo.tell() == off and self.fo.read() == '': return - raise FlowReadException("Invalid data format.") + raise exceptions.FlowReadException("Invalid data format.") class FilteredFlowWriter: @@ -79,5 +82,5 @@ def read_flows_from_paths(paths): with open(path, "rb") as f: flows.extend(FlowReader(f).stream()) except IOError as e: - raise FlowReadException(e.strerror) + raise exceptions.FlowReadException(e.strerror) return flows diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index e0d2c51e..a198018e 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -1,16 +1,22 @@ +from __future__ import absolute_import, print_function, division + import os import sys from typing import List, Optional, Set # noqa -from mitmproxy import controller, script, filt, models -from mitmproxy.exceptions import FlowReadException, Kill -from mitmproxy.flow import io, modules +import netlib.exceptions +from mitmproxy import controller +from mitmproxy import exceptions +from mitmproxy import filt +from mitmproxy import models +from mitmproxy import script +from mitmproxy.flow import io +from mitmproxy.flow import modules from mitmproxy.onboarding import app -from mitmproxy.protocol.http_replay import RequestReplayThread +from mitmproxy.protocol import http_replay from mitmproxy.proxy.config import HostMatcher from netlib import utils -from netlib.exceptions import HttpException class FlowMaster(controller.Master): @@ -311,7 +317,7 @@ class FlowMaster(controller.Master): freader = io.FlowReader(f) return self.load_flows(freader) except IOError as v: - raise FlowReadException(v.strerror) + raise exceptions.FlowReadException(v.strerror) def process_new_request(self, f): if self.stickycookie_state: @@ -351,7 +357,7 @@ class FlowMaster(controller.Master): f.response = None f.error = None self.process_new_request(f) - rt = RequestReplayThread( + rt = http_replay.RequestReplayThread( self.server.config, f, self.event_queue if run_scripthooks else False, @@ -405,7 +411,7 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(Kill) + f.reply(exceptions.Kill) return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) @@ -421,8 +427,8 @@ class FlowMaster(controller.Master): try: if self.stream_large_bodies: self.stream_large_bodies.run(f, False) - except HttpException: - f.reply(Kill) + except netlib.exceptions.HttpException: + f.reply(exceptions.Kill) return self.run_script_hook("responseheaders", f) return f diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index 438c5a21..601ebfce 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import, print_function, division + import collections import hashlib import re @@ -5,9 +7,12 @@ import re from six.moves import http_cookiejar from six.moves import urllib -from mitmproxy import version, filt, controller +from mitmproxy import controller +from mitmproxy import filt +from mitmproxy import version from netlib import wsgi -from netlib.http import http1, cookies +from netlib.http import cookies +from netlib.http import http1 class AppRegistry: diff --git a/mitmproxy/flow/state.py b/mitmproxy/flow/state.py index 16e79542..4287d77b 100644 --- a/mitmproxy/flow/state.py +++ b/mitmproxy/flow/state.py @@ -1,9 +1,12 @@ +from __future__ import absolute_import, print_function, division + from abc import abstractmethod, ABCMeta import six from typing import List # noqa -from mitmproxy import models, filt # noqa +from mitmproxy import filt +from mitmproxy import models # noqa @six.add_metaclass(ABCMeta) diff --git a/mitmproxy/main.py b/mitmproxy/main.py index d44c257e..b73966ba 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/main.py @@ -1,13 +1,16 @@ -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function, division + import os import signal import sys + from six.moves import _thread # PY3: We only need _thread.error, which is an alias of RuntimeError in 3.3+ -from netlib.version_check import check_pyopenssl_version -from . import cmdline -from .exceptions import ServerException -from .proxy.server import DummyServer, ProxyServer -from .proxy.config import process_proxy_options + +from mitmproxy import cmdline +from mitmproxy import exceptions +from mitmproxy.proxy import config +from mitmproxy.proxy import server +from netlib import version_check def assert_utf8_env(): @@ -28,11 +31,11 @@ def assert_utf8_env(): def get_server(dummy_server, options): if dummy_server: - return DummyServer(options) + return server.DummyServer(options) else: try: - return ProxyServer(options) - except ServerException as v: + return server.ProxyServer(options) + except exceptions.ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) @@ -44,7 +47,7 @@ def mitmproxy(args=None): # pragma: no cover sys.exit(1) from . import console - check_pyopenssl_version() + version_check.check_pyopenssl_version() assert_utf8_env() parser = cmdline.mitmproxy() @@ -52,7 +55,7 @@ def mitmproxy(args=None): # pragma: no cover if options.quiet: options.verbose = 0 - proxy_config = process_proxy_options(parser, options) + proxy_config = cmdline.process_proxy_options(parser, options) console_options = console.Options(**cmdline.get_common_options(options)) console_options.palette = options.palette console_options.palette_transparent = options.palette_transparent @@ -74,7 +77,7 @@ def mitmproxy(args=None): # pragma: no cover def mitmdump(args=None): # pragma: no cover from . import dump - check_pyopenssl_version() + version_check.check_pyopenssl_version() parser = cmdline.mitmdump() options = parser.parse_args(args) @@ -82,7 +85,7 @@ def mitmdump(args=None): # pragma: no cover options.verbose = 0 options.flow_detail = 0 - proxy_config = process_proxy_options(parser, options) + proxy_config = config.process_proxy_options(parser, options) dump_options = dump.Options(**cmdline.get_common_options(options)) dump_options.flow_detail = options.flow_detail dump_options.keepserving = options.keepserving @@ -108,7 +111,7 @@ def mitmdump(args=None): # pragma: no cover def mitmweb(args=None): # pragma: no cover from . import web - check_pyopenssl_version() + version_check.check_pyopenssl_version() parser = cmdline.mitmweb() @@ -116,7 +119,7 @@ def mitmweb(args=None): # pragma: no cover if options.quiet: options.verbose = 0 - proxy_config = process_proxy_options(parser, options) + proxy_config = config.process_proxy_options(parser, options) web_options = web.Options(**cmdline.get_common_options(options)) web_options.intercept = options.intercept web_options.wdebug = options.wdebug diff --git a/mitmproxy/models/__init__.py b/mitmproxy/models/__init__.py index 3d9d9dae..ca813567 100644 --- a/mitmproxy/models/__init__.py +++ b/mitmproxy/models/__init__.py @@ -1,12 +1,12 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division +from netlib.http import decoded +from .connections import ClientConnection, ServerConnection +from .flow import Flow, Error from .http import ( HTTPFlow, HTTPRequest, HTTPResponse, Headers, make_error_response, make_connect_request, make_connect_response, expect_continue_response ) -from netlib.http import decoded -from .connections import ClientConnection, ServerConnection -from .flow import Flow, Error from .tcp import TCPFlow FLOW_TYPES = dict( diff --git a/mitmproxy/models/connections.py b/mitmproxy/models/connections.py index 24cdff74..6347f488 100644 --- a/mitmproxy/models/connections.py +++ b/mitmproxy/models/connections.py @@ -1,12 +1,14 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division import copy import os import six -from netlib import tcp, certutils -from .. import stateobject, utils +from mitmproxy import stateobject +from mitmproxy import utils +from netlib import certutils +from netlib import tcp class ClientConnection(tcp.BaseHandler, stateobject.StateObject): diff --git a/mitmproxy/models/flow.py b/mitmproxy/models/flow.py index 8797fcd8..7b9ec030 100644 --- a/mitmproxy/models/flow.py +++ b/mitmproxy/models/flow.py @@ -1,10 +1,14 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import copy import uuid -from .. import stateobject, utils, version -from .connections import ClientConnection, ServerConnection -from ..exceptions import Kill +from mitmproxy import exceptions +from mitmproxy import stateobject +from mitmproxy import utils +from mitmproxy import version +from mitmproxy.models.connections import ClientConnection +from mitmproxy.models.connections import ServerConnection class Error(stateobject.StateObject): @@ -151,7 +155,7 @@ class Flow(stateobject.StateObject): """ self.error = Error("Connection killed") self.intercepted = False - self.reply(Kill) + self.reply(exceptions.Kill) master.error(self) def intercept(self, master): diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 43618ef8..a32124ac 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -1,11 +1,15 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import cgi +from mitmproxy import version +from mitmproxy.models.flow import Flow from netlib import encoding -from netlib.http import status_codes, Headers, Request, Response +from netlib.http import Headers +from netlib.http import Request +from netlib.http import Response +from netlib.http import status_codes from netlib.tcp import Address -from .. import version -from .flow import Flow class MessageMixin(object): diff --git a/mitmproxy/models/tcp.py b/mitmproxy/models/tcp.py index c7cfb9f8..e33475c2 100644 --- a/mitmproxy/models/tcp.py +++ b/mitmproxy/models/tcp.py @@ -1,8 +1,11 @@ +from __future__ import absolute_import, print_function, division + import time + from typing import List import netlib.basetypes -from .flow import Flow +from mitmproxy.models.flow import Flow class TCPMessage(netlib.basetypes.Serializable): diff --git a/mitmproxy/onboarding/app.py b/mitmproxy/onboarding/app.py index ff5ed63c..f93b9982 100644 --- a/mitmproxy/onboarding/app.py +++ b/mitmproxy/onboarding/app.py @@ -1,12 +1,13 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division + import os + +import tornado.template import tornado.web import tornado.wsgi -import tornado.template - -from .. import utils -from ..proxy import config +from mitmproxy import utils +from mitmproxy.proxy import config loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates")) diff --git a/mitmproxy/platform/osx.py b/mitmproxy/platform/osx.py index 3cd4bc66..b16c1861 100644 --- a/mitmproxy/platform/osx.py +++ b/mitmproxy/platform/osx.py @@ -1,4 +1,5 @@ import subprocess + import pf """ diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 0a810908..e160d050 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -1,18 +1,19 @@ -import configargparse -from six.moves import cPickle as pickle -from ctypes import byref, windll, Structure -from ctypes.wintypes import DWORD +import collections +import ctypes +import ctypes.wintypes import os import socket -from six.moves import socketserver import struct import threading import time -from collections import OrderedDict +import configargparse +from pydivert.enum import Direction +from pydivert.enum import Flag +from pydivert.enum import Layer from pydivert.windivert import WinDivert -from pydivert.enum import Direction, Layer, Flag - +from six.moves import cPickle as pickle +from six.moves import socketserver PROXY_API_PORT = 8085 @@ -91,22 +92,22 @@ ERROR_INSUFFICIENT_BUFFER = 0x7A # http://msdn.microsoft.com/en-us/library/windows/desktop/bb485761(v=vs.85).aspx -class MIB_TCPROW2(Structure): +class MIB_TCPROW2(ctypes.Structure): _fields_ = [ - ('dwState', DWORD), - ('dwLocalAddr', DWORD), - ('dwLocalPort', DWORD), - ('dwRemoteAddr', DWORD), - ('dwRemotePort', DWORD), - ('dwOwningPid', DWORD), - ('dwOffloadState', DWORD) + ('dwState', ctypes.wintypes.DWORD), + ('dwLocalAddr', ctypes.wintypes.DWORD), + ('dwLocalPort', ctypes.wintypes.DWORD), + ('dwRemoteAddr', ctypes.wintypes.DWORD), + ('dwRemotePort', ctypes.wintypes.DWORD), + ('dwOwningPid', ctypes.wintypes.DWORD), + ('dwOffloadState', ctypes.wintypes.DWORD) ] # http://msdn.microsoft.com/en-us/library/windows/desktop/bb485772(v=vs.85).aspx def MIB_TCPTABLE2(size): - class _MIB_TCPTABLE2(Structure): - _fields_ = [('dwNumEntries', DWORD), + class _MIB_TCPTABLE2(ctypes.Structure): + _fields_ = [('dwNumEntries', ctypes.wintypes.DWORD), ('table', MIB_TCPROW2 * size)] return _MIB_TCPTABLE2() @@ -192,7 +193,7 @@ class TransparentProxy(object): self.proxy_addr, self.proxy_port = proxy_addr, proxy_port self.connection_cache_size = cache_size - self.client_server_map = OrderedDict() + self.client_server_map = collections.OrderedDict() self.api = APIServer(self, (api_host, api_port), APIRequestHandler) self.api_thread = threading.Thread(target=self.api.serve_forever) @@ -212,7 +213,7 @@ class TransparentProxy(object): self.addr_pid_map = dict() self.trusted_pids = set() self.tcptable2 = MIB_TCPTABLE2(0) - self.tcptable2_size = DWORD(0) + self.tcptable2_size = ctypes.wintypes.DWORD(0) self.request_local_handle = None self.request_local_thread = threading.Thread(target=self.request_local) self.request_local_thread.daemon = True @@ -288,9 +289,9 @@ class TransparentProxy(object): raise def fetch_pids(self): - ret = windll.iphlpapi.GetTcpTable2( - byref( - self.tcptable2), byref( + ret = ctypes.windll.iphlpapi.GetTcpTable2( + ctypes.byref( + self.tcptable2), ctypes.byref( self.tcptable2_size), 0) if ret == ERROR_INSUFFICIENT_BUFFER: self.tcptable2 = MIB_TCPTABLE2(self.tcptable2_size.value) diff --git a/mitmproxy/protocol/__init__.py b/mitmproxy/protocol/__init__.py index 3d9fa7d4..510cd195 100644 --- a/mitmproxy/protocol/__init__.py +++ b/mitmproxy/protocol/__init__.py @@ -25,15 +25,16 @@ Another subtle design goal of this architecture is that upstream connections sho as late as possible; this makes server replay without any outgoing connections possible. """ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + from .base import Layer, ServerConnectionMixin -from .tls import TlsLayer -from .tls import is_tls_record_magic -from .tls import TlsClientHello from .http import UpstreamConnectLayer from .http1 import Http1Layer from .http2 import Http2Layer from .rawtcp import RawTCPLayer +from .tls import TlsClientHello +from .tls import TlsLayer +from .tls import is_tls_record_magic __all__ = [ "Layer", "ServerConnectionMixin", diff --git a/mitmproxy/protocol/base.py b/mitmproxy/protocol/base.py index c8e58d1b..11773385 100644 --- a/mitmproxy/protocol/base.py +++ b/mitmproxy/protocol/base.py @@ -1,11 +1,12 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import sys import six -from ..models import ServerConnection -from ..exceptions import ProtocolException -from netlib.exceptions import TcpException +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy import models class _LayerCodeCompletion(object): @@ -113,7 +114,7 @@ class ServerConnectionMixin(object): def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() - self.server_conn = ServerConnection(server_address, (self.config.host, 0)) + self.server_conn = models.ServerConnection(server_address, (self.config.host, 0)) self.__check_self_connect() def __check_self_connect(self): @@ -128,7 +129,7 @@ class ServerConnectionMixin(object): address.host in ("localhost", "127.0.0.1", "::1") ) if self_connect: - raise ProtocolException( + raise exceptions.ProtocolException( "Invalid server address: {}\r\n" "The proxy shall not connect to itself.".format(repr(address)) ) @@ -154,7 +155,7 @@ class ServerConnectionMixin(object): self.server_conn.finish() self.server_conn.close() self.channel.tell("serverdisconnect", self.server_conn) - self.server_conn = ServerConnection(address, (source_address.host, 0)) + self.server_conn = models.ServerConnection(address, (source_address.host, 0)) def connect(self): """ @@ -165,15 +166,15 @@ class ServerConnectionMixin(object): ~mitmproxy.exceptions.ProtocolException: if the connection could not be established. """ if not self.server_conn.address: - raise ProtocolException("Cannot connect to server, no server address given.") + raise exceptions.ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) self.channel.ask("serverconnect", self.server_conn) try: self.server_conn.connect() - except TcpException as e: + except netlib.exceptions.TcpException as e: six.reraise( - ProtocolException, - ProtocolException( + exceptions.ProtocolException, + exceptions.ProtocolException( "Server connection to {} failed: {}".format( repr(self.server_conn.address), str(e) ) diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index d9111303..eb740e3b 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -1,30 +1,21 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division import sys import traceback + +import h2.exceptions import six +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy import models +from mitmproxy import utils +from mitmproxy.protocol import base +from netlib import http from netlib import tcp -from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException -from netlib.http import Headers - -from h2.exceptions import H2Error - -from .. import utils -from ..exceptions import HttpProtocolException, Http2ProtocolException, ProtocolException -from ..models import ( - HTTPFlow, - HTTPResponse, - make_error_response, - make_connect_response, - Error, - expect_continue_response -) - -from .base import Layer -class _HttpTransmissionLayer(Layer): +class _HttpTransmissionLayer(base.Layer): def read_request(self): raise NotImplementedError() @@ -51,7 +42,7 @@ class _HttpTransmissionLayer(Layer): def send_response(self, response): if response.content is None: - raise HttpException("Cannot assemble flow with missing content") + raise netlib.exceptions.HttpException("Cannot assemble flow with missing content") self.send_response_headers(response) self.send_response_body(response, [response.content]) @@ -107,7 +98,7 @@ class UpstreamConnectLayer(Layer): self.send_request(self.connect_request) resp = self.read_response(self.connect_request) if resp.status_code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + raise exceptions.ProtocolException("Reconnect: Upstream server refuses CONNECT request") def connect(self): if not self.server_conn: @@ -166,16 +157,16 @@ class HttpLayer(Layer): self.handle_regular_mode_connect(request) return - except HttpReadDisconnect: + except netlib.exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return - except NetlibException as e: + except netlib.exceptions.NetlibException as e: self.send_error_response(400, repr(e)) - six.reraise(ProtocolException, ProtocolException( + six.reraise(exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self) flow.request = request # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: @@ -210,16 +201,16 @@ class HttpLayer(Layer): self.handle_upstream_mode_connect(flow.request.copy()) return - except (ProtocolException, NetlibException) as e: + except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not flow.response: - flow.error = Error(str(e)) + flow.error = models.Error(str(e)) self.channel.ask("error", flow) self.log(traceback.format_exc(), "debug") return else: - six.reraise(ProtocolException, ProtocolException( + six.reraise(exceptions.ProtocolException, exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: if flow: @@ -229,16 +220,16 @@ class HttpLayer(Layer): request = self.read_request() if request.headers.get("expect", "").lower() == "100-continue": # TODO: We may have to use send_response_headers for HTTP2 here. - self.send_response(expect_continue_response) + self.send_response(models.expect_continue_response) request.headers.pop("expect") request.body = b"".join(self.read_request_body(request)) return request def send_error_response(self, code, message): try: - response = make_error_response(code, message) + response = models.make_error_response(code, message) self.send_response(response) - except (NetlibException, H2Error, Http2ProtocolException): + except (netlib.exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException): self.log(traceback.format_exc(), "debug") def change_upstream_proxy_server(self, address): @@ -249,7 +240,7 @@ class HttpLayer(Layer): def handle_regular_mode_connect(self, request): self.set_server((request.host, request.port)) - self.send_response(make_connect_response(request.data.http_version)) + self.send_response(models.make_connect_response(request.data.http_version)) layer = self.ctx.next_layer(self) layer() @@ -283,7 +274,7 @@ class HttpLayer(Layer): try: get_response() - except NetlibException as e: + except netlib.exceptions.NetlibException as e: self.log( "server communication error: %s" % repr(e), level="debug" @@ -300,9 +291,9 @@ class HttpLayer(Layer): # > read (100-n)% of large request # > send large request upstream - if isinstance(e, Http2ProtocolException): + if isinstance(e, exceptions.Http2ProtocolException): # do not try to reconnect for HTTP2 - raise ProtocolException("First and only attempt to get response via HTTP2 failed.") + raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.") self.disconnect() self.connect() @@ -345,7 +336,7 @@ class HttpLayer(Layer): flow.request.scheme = "https" if self.__initial_server_tls else "http" request_reply = self.channel.ask("request", flow) - if isinstance(request_reply, HTTPResponse): + if isinstance(request_reply, models.HTTPResponse): flow.response = request_reply return @@ -365,7 +356,7 @@ class HttpLayer(Layer): if not self.server_conn: self.connect() if tls: - raise HttpProtocolException("Cannot change scheme in upstream proxy mode.") + raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: @@ -383,7 +374,7 @@ class HttpLayer(Layer): def validate_request(self, request): if request.first_line_format == "absolute" and request.scheme != "http": - raise HttpException("Invalid request scheme: %s" % request.scheme) + raise netlib.exceptions.HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { "regular": ("authority", "absolute",), @@ -396,7 +387,7 @@ class HttpLayer(Layer): err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( " or ".join(allowed_request_forms), request.first_line_format ) - raise HttpException(err_message) + raise netlib.exceptions.HttpException(err_message) if self.mode == "regular" and request.first_line_format == "absolute": request.first_line_format = "relative" @@ -406,10 +397,10 @@ class HttpLayer(Layer): if self.config.authenticator.authenticate(request.headers): self.config.authenticator.clean(request.headers) else: - self.send_response(make_error_response( + self.send_response(models.make_error_response( 407, "Proxy Authentication Required", - Headers(**self.config.authenticator.auth_challenge_headers()) + http.Headers(**self.config.authenticator.auth_challenge_headers()) )) return False return True diff --git a/mitmproxy/protocol/http1.py b/mitmproxy/protocol/http1.py index 940a4c98..7055a7fd 100644 --- a/mitmproxy/protocol/http1.py +++ b/mitmproxy/protocol/http1.py @@ -1,13 +1,11 @@ -from __future__ import (absolute_import, print_function, division) - +from __future__ import absolute_import, print_function, division +from mitmproxy import models +from mitmproxy.protocol import http from netlib.http import http1 -from .http import _HttpTransmissionLayer, HttpLayer -from ..models import HTTPRequest, HTTPResponse - -class Http1Layer(_HttpTransmissionLayer): +class Http1Layer(http._HttpTransmissionLayer): def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) @@ -15,7 +13,7 @@ class Http1Layer(_HttpTransmissionLayer): def read_request(self): req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit) - return HTTPRequest.wrap(req) + return models.HTTPRequest.wrap(req) def read_request_body(self, request): expected_size = http1.expected_http_body_size(request) @@ -27,7 +25,7 @@ class Http1Layer(_HttpTransmissionLayer): def read_response_headers(self): resp = http1.read_response_head(self.server_conn.rfile) - return HTTPResponse.wrap(resp) + return models.HTTPResponse.wrap(resp) def read_response_body(self, request, response): expected_size = http1.expected_http_body_size(request, response) @@ -63,5 +61,5 @@ class Http1Layer(_HttpTransmissionLayer): return close_connection def __call__(self): - layer = HttpLayer(self, self.mode) + layer = http.HttpLayer(self, self.mode) layer() diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 6ca78347..0fa4d77b 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -1,29 +1,27 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division import threading import time -from six.moves import queue - import traceback + +import h2.exceptions +import hyperframe import six -from h2.connection import H2Connection -from h2.exceptions import StreamClosedError +from h2 import connection from h2 import events -from hyperframe.frame import PriorityFrame - -from netlib.tcp import ssl_read_select -from netlib.exceptions import HttpException -from netlib.http import Headers -from netlib.http.http2 import framereader -import netlib.http.url +from six.moves import queue -from .base import Layer -from .http import _HttpTransmissionLayer, HttpLayer -from ..exceptions import ProtocolException, Http2ProtocolException -from ..models import HTTPRequest, HTTPResponse +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy import models +from mitmproxy import protocol +from netlib import http +from netlib import tcp +from netlib.http import http2 +from netlib.http import url -class SafeH2Connection(H2Connection): +class SafeH2Connection(connection.H2Connection): def __init__(self, conn, *args, **kwargs): super(SafeH2Connection, self).__init__(*args, **kwargs) @@ -46,7 +44,7 @@ class SafeH2Connection(H2Connection): with self.lock: try: self.reset_stream(stream_id, error_code) - except StreamClosedError: # pragma: no cover + except h2.exceptions.StreamClosedError: # pragma: no cover # stream is already closed - good pass self.conn.send(self.data_to_send()) @@ -59,7 +57,7 @@ class SafeH2Connection(H2Connection): def safe_send_headers(self, is_zombie, stream_id, headers): with self.lock: if is_zombie(): # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") self.send_headers(stream_id, headers.fields) self.conn.send(self.data_to_send()) @@ -70,7 +68,7 @@ class SafeH2Connection(H2Connection): self.lock.acquire() if is_zombie(): # pragma: no cover self.lock.release() - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") max_outbound_frame_size = self.max_outbound_frame_size frame_chunk = chunk[position:position + max_outbound_frame_size] if self.local_flow_control_window(stream_id) < len(frame_chunk): @@ -83,12 +81,12 @@ class SafeH2Connection(H2Connection): position += max_outbound_frame_size with self.lock: if is_zombie(): # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") self.end_stream(stream_id) self.conn.send(self.data_to_send()) -class Http2Layer(Layer): +class Http2Layer(protocol.base.Layer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) @@ -108,13 +106,13 @@ class Http2Layer(Layer): self.active_conns.append(self.server_conn.connection) def connect(self): # pragma: no cover - raise Http2ProtocolException("HTTP2 layer should already have a connection.") + raise exceptions.Http2ProtocolException("HTTP2 layer should already have a connection.") def set_server(self): # pragma: no cover - raise Http2ProtocolException("Cannot change server for HTTP2 connections.") + raise exceptions.Http2ProtocolException("Cannot change server for HTTP2 connections.") def disconnect(self): # pragma: no cover - raise Http2ProtocolException("Cannot dis- or reconnect in HTTP2 connections.") + raise exceptions.Http2ProtocolException("Cannot dis- or reconnect in HTTP2 connections.") def next_layer(self): # pragma: no cover # WebSockets over HTTP/2? @@ -135,19 +133,19 @@ class Http2Layer(Layer): eid = event.stream_id if isinstance(event, events.RequestReceived): - headers = Headers([[k, v] for k, v in event.headers]) + headers = http.Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, eid, headers) self.streams[eid].timestamp_start = time.time() self.streams[eid].start() elif isinstance(event, events.ResponseReceived): - headers = Headers([[k, v] for k, v in event.headers]) + headers = http.Headers([[k, v] for k, v in event.headers]) self.streams[eid].queued_data_length = 0 self.streams[eid].timestamp_start = time.time() self.streams[eid].response_headers = headers self.streams[eid].response_arrived.set() elif isinstance(event, events.DataReceived): if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit: - raise HttpException("HTTP body too large. Limit is {}.".format(self.config.body_size_limit)) + raise netlib.exceptions.HttpException("HTTP body too large. Limit is {}.".format(self.config.body_size_limit)) self.streams[eid].data_queue.put(event.data) self.streams[eid].queued_data_length += len(event.data) source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length) @@ -178,7 +176,7 @@ class Http2Layer(Layer): self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers) self.client_conn.send(self.client_conn.h2.data_to_send()) - headers = Headers([[str(k), str(v)] for k, v in event.headers]) + headers = http.Headers([[str(k), str(v)] for k, v in event.headers]) headers['x-mitmproxy-pushed'] = 'true' self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, event.pushed_stream_id, headers) self.streams[event.pushed_stream_id].timestamp_start = time.time() @@ -197,7 +195,7 @@ class Http2Layer(Layer): depends_on = self.streams[depends_on].server_stream_id # weight is between 1 and 256 (inclusive), but represented as uint8 (0 to 255) - frame = PriorityFrame(stream_id, depends_on, event.weight - 1, event.exclusive) + frame = hyperframe.frame.PriorityFrame(stream_id, depends_on, event.weight - 1, event.exclusive) self.server_conn.send(frame.serialize()) elif isinstance(event, events.TrailersReceived): raise NotImplementedError() @@ -226,7 +224,7 @@ class Http2Layer(Layer): self.client_conn.send(self.client_conn.h2.data_to_send()) while True: - r = ssl_read_select(self.active_conns, 1) + r = tcp.ssl_read_select(self.active_conns, 1) for conn in r: source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn @@ -234,7 +232,7 @@ class Http2Layer(Layer): with source_conn.h2.lock: try: - raw_frame = b''.join(framereader.http2_read_raw_frame(source_conn.rfile)) + raw_frame = b''.join(http2.framereader.http2_read_raw_frame(source_conn.rfile)) except: # read frame failed: connection closed self._kill_all_streams() @@ -252,7 +250,7 @@ class Http2Layer(Layer): self._cleanup_streams() -class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): +class Http2SingleStreamLayer(protocol.http._HttpTransmissionLayer, threading.Thread): def __init__(self, ctx, stream_id, request_headers): super(Http2SingleStreamLayer, self).__init__(ctx, name="Thread-Http2SingleStreamLayer-{}".format(stream_id)) @@ -336,7 +334,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): data.append(self.request_data_queue.get()) data = b"".join(data) - return HTTPRequest( + return models.HTTPRequest( first_line_format, method, scheme, @@ -361,7 +359,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): with self.server_conn.h2.lock: # We must not assign a stream id if we are already a zombie. if self.zombie: # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") self.server_stream_id = self.server_conn.h2.get_next_available_stream_id() self.server_to_client_stream_ids[self.server_stream_id] = self.client_stream_id @@ -382,7 +380,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): message.body ) if self.zombie: # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") def read_response_headers(self): self.response_arrived.wait() @@ -391,7 +389,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): headers = self.response_headers.copy() headers.clear(":status") - return HTTPResponse( + return models.HTTPResponse( http_version=b"HTTP/2.0", status_code=status_code, reason='', @@ -412,7 +410,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): yield self.response_data_queue.get() return if self.zombie: # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") def send_response_headers(self, response): headers = response.headers.copy() @@ -423,7 +421,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): headers ) if self.zombie: # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") def send_response_body(self, _response, chunks): self.client_conn.h2.safe_send_body( @@ -432,7 +430,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): chunks ) if self.zombie: # pragma: no cover - raise Http2ProtocolException("Zombie Stream") + raise exceptions.Http2ProtocolException("Zombie Stream") def check_close_connection(self, flow): # This layer only handles a single stream. @@ -447,11 +445,11 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread): self() def __call__(self): - layer = HttpLayer(self, self.mode) + layer = protocol.http.HttpLayer(self, self.mode) try: layer() - except ProtocolException as e: + except exceptions.ProtocolException as e: self.log(repr(e), "info") self.log(traceback.format_exc(), "debug") diff --git a/mitmproxy/protocol/http_replay.py b/mitmproxy/protocol/http_replay.py index f7f27148..5928c0af 100644 --- a/mitmproxy/protocol/http_replay.py +++ b/mitmproxy/protocol/http_replay.py @@ -1,13 +1,14 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import threading import traceback -from mitmproxy.exceptions import ReplayException -from netlib.exceptions import HttpException, TcpException + +import netlib.exceptions +from mitmproxy import controller +from mitmproxy import exceptions +from mitmproxy import models from netlib.http import http1 -from ..controller import Channel -from ..models import Error, HTTPResponse, ServerConnection, make_connect_request -from ..exceptions import Kill # TODO: Doesn't really belong into mitmproxy.protocol... @@ -22,7 +23,7 @@ class RequestReplayThread(threading.Thread): """ self.config, self.flow = config, flow if event_queue: - self.channel = Channel(event_queue, should_exit) + self.channel = controller.Channel(event_queue, should_exit) else: self.channel = None super(RequestReplayThread, self).__init__() @@ -36,17 +37,17 @@ class RequestReplayThread(threading.Thread): # If we have a channel, run script hooks. if self.channel: request_reply = self.channel.ask("request", self.flow) - if isinstance(request_reply, HTTPResponse): + if isinstance(request_reply, models.HTTPResponse): self.flow.response = request_reply if not self.flow.response: # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": server_address = self.config.upstream_server.address - server = ServerConnection(server_address, (self.config.host, 0)) + server = models.ServerConnection(server_address, (self.config.host, 0)) server.connect() if r.scheme == "https": - connect_request = make_connect_request((r.host, r.port)) + connect_request = models.make_connect_request((r.host, r.port)) server.wfile.write(http1.assemble_request(connect_request)) server.wfile.flush() resp = http1.read_response( @@ -55,7 +56,7 @@ class RequestReplayThread(threading.Thread): body_size_limit=self.config.body_size_limit ) if resp.status_code != 200: - raise ReplayException("Upstream server refuses CONNECT request") + raise exceptions.ReplayException("Upstream server refuses CONNECT request") server.establish_ssl( self.config.clientcerts, sni=self.flow.server_conn.sni @@ -65,7 +66,7 @@ class RequestReplayThread(threading.Thread): r.first_line_format = "absolute" else: server_address = (r.host, r.port) - server = ServerConnection(server_address, (self.config.host, 0)) + server = models.ServerConnection(server_address, (self.config.host, 0)) server.connect() if r.scheme == "https": server.establish_ssl( @@ -77,20 +78,20 @@ class RequestReplayThread(threading.Thread): server.wfile.write(http1.assemble_request(r)) server.wfile.flush() self.flow.server_conn = server - self.flow.response = HTTPResponse.wrap(http1.read_response( + self.flow.response = models.HTTPResponse.wrap(http1.read_response( server.rfile, r, body_size_limit=self.config.body_size_limit )) if self.channel: response_reply = self.channel.ask("response", self.flow) - if response_reply == Kill: - raise Kill() - except (ReplayException, HttpException, TcpException) as e: - self.flow.error = Error(str(e)) + if response_reply == exceptions.Kill: + raise exceptions.Kill() + except (exceptions.ReplayException, netlib.exceptions.NetlibException) as e: + self.flow.error = models.Error(str(e)) if self.channel: self.channel.ask("error", self.flow) - except Kill: + except exceptions.Kill: # Kill should only be raised if there's a channel in the # first place. from ..proxy.root_context import Log diff --git a/mitmproxy/protocol/rawtcp.py b/mitmproxy/protocol/rawtcp.py index 05342844..15f145e2 100644 --- a/mitmproxy/protocol/rawtcp.py +++ b/mitmproxy/protocol/rawtcp.py @@ -1,17 +1,16 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import socket from OpenSSL import SSL -from netlib.exceptions import TcpException - -from netlib.tcp import ssl_read_select -from ..models import Error -from ..models.tcp import TCPFlow, TCPMessage -from .base import Layer +import netlib.exceptions +from mitmproxy import models +from mitmproxy.protocol import base +from netlib import tcp -class RawTCPLayer(Layer): +class RawTCPLayer(base.Layer): chunk_size = 4096 def __init__(self, ctx, ignore=False): @@ -22,7 +21,7 @@ class RawTCPLayer(Layer): self.connect() if not self.ignore: - flow = TCPFlow(self.client_conn, self.server_conn, self) + flow = models.TCPFlow(self.client_conn, self.server_conn, self) self.channel.ask("tcp_open", flow) buf = memoryview(bytearray(self.chunk_size)) @@ -33,7 +32,7 @@ class RawTCPLayer(Layer): try: while not self.channel.should_exit.is_set(): - r = ssl_read_select(conns, 10) + r = tcp.ssl_read_select(conns, 10) for conn in r: dst = server if conn == client else client @@ -52,15 +51,15 @@ class RawTCPLayer(Layer): return continue - tcp_message = TCPMessage(dst == server, buf[:size].tobytes()) + tcp_message = models.TCPMessage(dst == server, buf[:size].tobytes()) if not self.ignore: flow.messages.append(tcp_message) self.channel.ask("tcp_message", flow) dst.sendall(tcp_message.content) - except (socket.error, TcpException, SSL.Error) as e: + except (socket.error, netlib.exceptions.TcpException, SSL.Error) as e: if not self.ignore: - flow.error = Error("TCP connection closed unexpectedly: {}".format(repr(e))) + flow.error = models.Error("TCP connection closed unexpectedly: {}".format(repr(e))) self.channel.tell("tcp_error", flow) finally: if not self.ignore: diff --git a/mitmproxy/protocol/tls.py b/mitmproxy/protocol/tls.py index e0699562..0bbb2cc2 100644 --- a/mitmproxy/protocol/tls.py +++ b/mitmproxy/protocol/tls.py @@ -1,16 +1,15 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division import struct import sys -from construct import ConstructError +import construct import six -from netlib.exceptions import InvalidCertificateException -from netlib.exceptions import TlsException -from ..contrib.tls._constructs import ClientHello -from ..exceptions import ProtocolException, TlsProtocolException, ClientHandshakeException -from .base import Layer +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy.contrib.tls import _constructs +from mitmproxy.protocol import base # taken from https://testssl.sh/openssl-rfc.mappping.html @@ -246,11 +245,11 @@ def get_client_hello(client_conn): while len(client_hello) < client_hello_size: record_header = client_conn.rfile.peek(offset + 5)[offset:] if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header) + raise exceptions.TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header) record_size = struct.unpack("!H", record_header[3:])[0] + 5 record_body = client_conn.rfile.peek(offset + record_size)[offset + 5:] if len(record_body) != record_size - 5: - raise TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + raise exceptions.TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4 @@ -260,7 +259,7 @@ def get_client_hello(client_conn): class TlsClientHello(object): def __init__(self, raw_client_hello): - self._client_hello = ClientHello.parse(raw_client_hello) + self._client_hello = _constructs.ClientHello.parse(raw_client_hello) def raw(self): return self._client_hello @@ -297,13 +296,13 @@ class TlsClientHello(object): """ try: raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header. - except ProtocolException as e: - raise TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) + except exceptions.ProtocolException as e: + raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) try: return cls(raw_client_hello) - except ConstructError as e: - raise TlsProtocolException('Cannot parse Client Hello: %s, Raw Client Hello: %s' % + except construct.ConstructError as e: + raise exceptions.TlsProtocolException('Cannot parse Client Hello: %s, Raw Client Hello: %s' % (repr(e), raw_client_hello.encode("hex"))) def __repr__(self): @@ -311,7 +310,7 @@ class TlsClientHello(object): (self.sni, self.alpn_protocols, self.cipher_suites) -class TlsLayer(Layer): +class TlsLayer(base.Layer): """ The TLS layer implements transparent TLS connections. @@ -345,7 +344,7 @@ class TlsLayer(Layer): # Peek into the connection, read the initial client hello and parse it to obtain SNI and ALPN values. try: self._client_hello = TlsClientHello.from_client_conn(self.client_conn) - except TlsProtocolException as e: + except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") # Do we need to do a server handshake now? @@ -490,10 +489,10 @@ class TlsLayer(Layer): # The reason for this might be difficult to find, so we try to peek here to see if it # raises ann error. self.client_conn.rfile.peek(1) - except TlsException as e: + except netlib.exceptions.TlsException as e: six.reraise( - ClientHandshakeException, - ClientHandshakeException( + exceptions.ClientHandshakeException, + exceptions.ClientHandshakeException( "Cannot establish TLS with client (sni: {sni}): {e}".format( sni=self._client_hello.sni, e=repr(e) ), @@ -544,7 +543,7 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Ignoring server verification error, continuing with connection", "error") - except InvalidCertificateException as e: + except netlib.exceptions.InvalidCertificateException as e: tls_cert_err = self.server_conn.ssl_verification_error self.log( "TLS verification failed for upstream server at depth %s with error: %s" % @@ -552,18 +551,18 @@ class TlsLayer(Layer): "error") self.log("Aborting connection attempt", "error") six.reraise( - TlsProtocolException, - TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + exceptions.TlsProtocolException, + exceptions.TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( address=repr(self.server_conn.address), sni=self.server_sni, e=repr(e), )), sys.exc_info()[2] ) - except TlsException as e: + except netlib.exceptions.TlsException as e: six.reraise( - TlsProtocolException, - TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + exceptions.TlsProtocolException, + exceptions.TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( address=repr(self.server_conn.address), sni=self.server_sni, e=repr(e), diff --git a/mitmproxy/proxy/__init__.py b/mitmproxy/proxy/__init__.py index be7f5207..ada9fa12 100644 --- a/mitmproxy/proxy/__init__.py +++ b/mitmproxy/proxy/__init__.py @@ -1,8 +1,8 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division -from .server import ProxyServer, DummyServer from .config import ProxyConfig from .root_context import RootContext, Log +from .server import ProxyServer, DummyServer __all__ = [ "ProxyServer", "DummyServer", diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 163e92dd..9246fe04 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -1,4 +1,5 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import collections import os import re @@ -6,11 +7,11 @@ import re import six from OpenSSL import SSL -from netlib import certutils, tcp, human +from mitmproxy import platform +from netlib import certutils +from netlib import human +from netlib import tcp from netlib.http import authentication -from netlib.tcp import Address, sslversion_choices - -from .. import platform CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -91,7 +92,7 @@ class ProxyConfig: self.body_size_limit = body_size_limit self.mode = mode if upstream_server: - self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) + self.upstream_server = ServerSpec(upstream_server[0], tcp.Address.wrap(upstream_server[1])) self.upstream_auth = upstream_auth else: self.upstream_server = None @@ -111,9 +112,9 @@ class ProxyConfig: self.certstore.add_cert_file(spec, cert) self.openssl_method_client, self.openssl_options_client = \ - sslversion_choices[ssl_version_client] + tcp.sslversion_choices[ssl_version_client] self.openssl_method_server, self.openssl_options_server = \ - sslversion_choices[ssl_version_server] + tcp.sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER diff --git a/mitmproxy/proxy/modes/__init__.py b/mitmproxy/proxy/modes/__init__.py index f014ed98..fa62570c 100644 --- a/mitmproxy/proxy/modes/__init__.py +++ b/mitmproxy/proxy/modes/__init__.py @@ -1,4 +1,5 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + from .http_proxy import HttpProxy, HttpUpstreamProxy from .reverse_proxy import ReverseProxy from .socks_proxy import Socks5Proxy diff --git a/mitmproxy/proxy/modes/http_proxy.py b/mitmproxy/proxy/modes/http_proxy.py index e19062b9..bc64ccd2 100644 --- a/mitmproxy/proxy/modes/http_proxy.py +++ b/mitmproxy/proxy/modes/http_proxy.py @@ -1,9 +1,9 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division -from ...protocol import Layer, ServerConnectionMixin +from mitmproxy import protocol -class HttpProxy(Layer, ServerConnectionMixin): +class HttpProxy(protocol.Layer, protocol.ServerConnectionMixin): def __call__(self): layer = self.ctx.next_layer(self) @@ -14,7 +14,7 @@ class HttpProxy(Layer, ServerConnectionMixin): self.disconnect() -class HttpUpstreamProxy(Layer, ServerConnectionMixin): +class HttpUpstreamProxy(protocol.Layer, protocol.ServerConnectionMixin): def __init__(self, ctx, server_address): super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) diff --git a/mitmproxy/proxy/modes/reverse_proxy.py b/mitmproxy/proxy/modes/reverse_proxy.py index c8e80a10..3739ac0e 100644 --- a/mitmproxy/proxy/modes/reverse_proxy.py +++ b/mitmproxy/proxy/modes/reverse_proxy.py @@ -1,9 +1,9 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division -from ...protocol import Layer, ServerConnectionMixin +from mitmproxy import protocol -class ReverseProxy(Layer, ServerConnectionMixin): +class ReverseProxy(protocol.Layer, protocol.ServerConnectionMixin): def __init__(self, ctx, server_address, server_tls): super(ReverseProxy, self).__init__(ctx, server_address=server_address) diff --git a/mitmproxy/proxy/modes/socks_proxy.py b/mitmproxy/proxy/modes/socks_proxy.py index e2ce44ae..19437835 100644 --- a/mitmproxy/proxy/modes/socks_proxy.py +++ b/mitmproxy/proxy/modes/socks_proxy.py @@ -1,13 +1,13 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division -from netlib import socks, tcp -from netlib.exceptions import TcpException +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy import protocol +from netlib import socks +from netlib import tcp -from ...exceptions import Socks5ProtocolException -from ...protocol import Layer, ServerConnectionMixin - -class Socks5Proxy(Layer, ServerConnectionMixin): +class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin): def __init__(self, ctx): super(Socks5Proxy, self).__init__(ctx) @@ -51,8 +51,8 @@ class Socks5Proxy(Layer, ServerConnectionMixin): connect_reply.to_file(self.client_conn.wfile) self.client_conn.wfile.flush() - except (socks.SocksError, TcpException) as e: - raise Socks5ProtocolException("SOCKS5 mode failure: %s" % repr(e)) + except (socks.SocksError, netlib.exceptions.TcpException) as e: + raise exceptions.Socks5ProtocolException("SOCKS5 mode failure: %s" % repr(e)) # https://github.com/mitmproxy/mitmproxy/issues/839 address_bytes = (connect_request.addr.host.encode("idna"), connect_request.addr.port) diff --git a/mitmproxy/proxy/modes/transparent_proxy.py b/mitmproxy/proxy/modes/transparent_proxy.py index 3fdda656..c7df7900 100644 --- a/mitmproxy/proxy/modes/transparent_proxy.py +++ b/mitmproxy/proxy/modes/transparent_proxy.py @@ -1,11 +1,11 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division -from ... import platform -from ...exceptions import ProtocolException -from ...protocol import Layer, ServerConnectionMixin +from mitmproxy import exceptions +from mitmproxy import platform +from mitmproxy import protocol -class TransparentProxy(Layer, ServerConnectionMixin): +class TransparentProxy(protocol.Layer, protocol.ServerConnectionMixin): def __init__(self, ctx): super(TransparentProxy, self).__init__(ctx) @@ -15,7 +15,7 @@ class TransparentProxy(Layer, ServerConnectionMixin): try: self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) except Exception as e: - raise ProtocolException("Transparent mode failure: %s" % repr(e)) + raise exceptions.ProtocolException("Transparent mode failure: %s" % repr(e)) layer = self.ctx.next_layer(self) try: diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index 9b4e2963..6a4162df 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -1,15 +1,13 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import sys import six -from mitmproxy.exceptions import ProtocolException, TlsProtocolException -from netlib.exceptions import TcpException -from ..protocol import ( - RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin, - UpstreamConnectLayer, TlsClientHello -) -from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy import protocol +from mitmproxy.proxy import modes class RootContext(object): @@ -50,53 +48,53 @@ class RootContext(object): def _next_layer(self, top_layer): try: d = top_layer.client_conn.rfile.peek(3) - except TcpException as e: - six.reraise(ProtocolException, ProtocolException(str(e)), sys.exc_info()[2]) - client_tls = is_tls_record_magic(d) + except netlib.exceptions.TcpException as e: + six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(str(e)), sys.exc_info()[2]) + client_tls = protocol.is_tls_record_magic(d) # 1. check for --ignore if self.config.check_ignore: ignore = self.config.check_ignore(top_layer.server_conn.address) if not ignore and client_tls: try: - client_hello = TlsClientHello.from_client_conn(self.client_conn) - except TlsProtocolException as e: + client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn) + except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: ignore = self.config.check_ignore((client_hello.sni, 443)) if ignore: - return RawTCPLayer(top_layer, ignore=True) + return protocol.RawTCPLayer(top_layer, ignore=True) # 2. Always insert a TLS layer, even if there's neither client nor server tls. # An inline script may upgrade from http to https, # in which case we need some form of TLS layer. - if isinstance(top_layer, ReverseProxy): - return TlsLayer(top_layer, client_tls, top_layer.server_tls) - if isinstance(top_layer, ServerConnectionMixin) or isinstance(top_layer, UpstreamConnectLayer): - return TlsLayer(top_layer, client_tls, client_tls) + if isinstance(top_layer, modes.ReverseProxy): + return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer): + return protocol.TlsLayer(top_layer, client_tls, client_tls) # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. - if isinstance(top_layer, TlsLayer): - if isinstance(top_layer.ctx, HttpProxy): - return Http1Layer(top_layer, "regular") - if isinstance(top_layer.ctx, HttpUpstreamProxy): - return Http1Layer(top_layer, "upstream") + if isinstance(top_layer, protocol.TlsLayer): + if isinstance(top_layer.ctx, modes.HttpProxy): + return protocol.Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, modes.HttpUpstreamProxy): + return protocol.Http1Layer(top_layer, "upstream") # 4. Check for other TLS cases (e.g. after CONNECT). if client_tls: - return TlsLayer(top_layer, True, True) + return protocol.TlsLayer(top_layer, True, True) # 4. Check for --tcp if self.config.check_tcp(top_layer.server_conn.address): - return RawTCPLayer(top_layer) + return protocol.RawTCPLayer(top_layer) # 5. Check for TLS ALPN (HTTP1/HTTP2) - if isinstance(top_layer, TlsLayer): + if isinstance(top_layer, protocol.TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() if alpn == b'h2': - return Http2Layer(top_layer, 'transparent') + return protocol.Http2Layer(top_layer, 'transparent') if alpn == b'http/1.1': - return Http1Layer(top_layer, 'transparent') + return protocol.Http1Layer(top_layer, 'transparent') # 6. Check for raw tcp mode is_ascii = ( @@ -105,10 +103,10 @@ class RootContext(object): all(65 <= x <= 90 and 97 <= x <= 122 for x in six.iterbytes(d)) ) if self.config.rawtcp and not is_ascii: - return RawTCPLayer(top_layer) + return protocol.RawTCPLayer(top_layer) # 7. Assume HTTP1 by default - return Http1Layer(top_layer, 'transparent') + return protocol.Http1Layer(top_layer, 'transparent') def log(self, msg, level, subs=()): """ diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 8483d3df..933d043d 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -1,18 +1,18 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division -import traceback -import sys import socket +import sys +import traceback + import six +import netlib.exceptions +from mitmproxy import exceptions +from mitmproxy import models +from mitmproxy.proxy import modes +from mitmproxy.proxy import root_context from netlib import tcp -from netlib.exceptions import TcpException -from netlib.http.http1 import assemble_response -from ..exceptions import ProtocolException, ServerException, ClientHandshakeException, Kill -from ..models import ClientConnection, make_error_response -from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy -from .root_context import RootContext, Log - +from netlib.http import http1 class DummyServer: bound = False @@ -43,8 +43,8 @@ class ProxyServer(tcp.TCPServer): super(ProxyServer, self).__init__((config.host, config.port)) except socket.error as e: six.reraise( - ServerException, - ServerException('Error starting proxy server: ' + repr(e)), + exceptions.ServerException, + exceptions.ServerException('Error starting proxy server: ' + repr(e)), sys.exc_info()[2] ) self.channel = None @@ -67,7 +67,7 @@ class ConnectionHandler(object): def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: mitmproxy.proxy.config.ProxyConfig""" - self.client_conn = ClientConnection( + self.client_conn = models.ClientConnection( client_conn, client_address, None) @@ -76,7 +76,7 @@ class ConnectionHandler(object): """@type: mitmproxy.controller.Channel""" def _create_root_layer(self): - root_context = RootContext( + root_ctx = root_context.RootContext( self.client_conn, self.config, self.channel @@ -84,25 +84,25 @@ class ConnectionHandler(object): mode = self.config.mode if mode == "upstream": - return HttpUpstreamProxy( - root_context, + return modes.HttpUpstreamProxy( + root_ctx, self.config.upstream_server.address ) elif mode == "transparent": - return TransparentProxy(root_context) + return modes.TransparentProxy(root_ctx) elif mode == "reverse": server_tls = self.config.upstream_server.scheme == "https" - return ReverseProxy( - root_context, + return modes.ReverseProxy( + root_ctx, self.config.upstream_server.address, server_tls ) elif mode == "socks5": - return Socks5Proxy(root_context) + return modes.Socks5Proxy(root_ctx) elif mode == "regular": - return HttpProxy(root_context) + return modes.HttpProxy(root_ctx) elif callable(mode): # pragma: no cover - return mode(root_context) + return mode(root_ctx) else: # pragma: no cover raise ValueError("Unknown proxy mode: %s" % mode) @@ -114,11 +114,11 @@ class ConnectionHandler(object): try: root_layer = self.channel.ask("clientconnect", root_layer) root_layer() - except Kill: + except exceptions.Kill: self.log("Connection killed", "info") - except ProtocolException as e: + except exceptions.ProtocolException as e: - if isinstance(e, ClientHandshakeException): + if isinstance(e, exceptions.ClientHandshakeException): self.log( "Client Handshake failed. " "The client may not trust the proxy's certificate for {}.".format(e.server), @@ -133,9 +133,9 @@ class ConnectionHandler(object): # we send an HTTP error response, which is both # understandable by HTTP clients and humans. try: - error_response = make_error_response(502, repr(e)) - self.client_conn.send(assemble_response(error_response)) - except TcpException: + error_response = models.make_error_response(502, repr(e)) + self.client_conn.send(http1.assemble_response(error_response)) + except netlib.exceptions.TcpException: pass except Exception: self.log(traceback.format_exc(), "error") @@ -149,4 +149,4 @@ class ConnectionHandler(object): def log(self, msg, level): msg = "{}: {}".format(repr(self.client_conn.address), msg) - self.channel.tell("log", Log(msg, level)) + self.channel.tell("log", root_context.Log(msg, level)) diff --git a/mitmproxy/script/__init__.py b/mitmproxy/script/__init__.py index 3ee19b04..d6bff4c7 100644 --- a/mitmproxy/script/__init__.py +++ b/mitmproxy/script/__init__.py @@ -1,8 +1,8 @@ +from . import reloader +from .concurrent import concurrent from .script import Script from .script_context import ScriptContext -from .concurrent import concurrent from ..exceptions import ScriptException -from . import reloader __all__ = [ "Script", diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py index 2f25e78c..43d0d328 100644 --- a/mitmproxy/script/concurrent.py +++ b/mitmproxy/script/concurrent.py @@ -3,6 +3,7 @@ This module provides a @concurrent decorator primitive to offload computations from mitmproxy's main master thread. """ from __future__ import absolute_import, print_function, division + import threading diff --git a/mitmproxy/script/reloader.py b/mitmproxy/script/reloader.py index 99ce7f60..f5470bbf 100644 --- a/mitmproxy/script/reloader.py +++ b/mitmproxy/script/reloader.py @@ -1,6 +1,10 @@ +from __future__ import absolute_import, print_function, division + import os import sys + from watchdog.events import RegexMatchingEventHandler + if sys.platform == 'darwin': # pragma: no cover from watchdog.observers.polling import PollingObserver as Observer else: diff --git a/mitmproxy/script/script.py b/mitmproxy/script/script.py index 9bf05f27..70f74817 100644 --- a/mitmproxy/script/script.py +++ b/mitmproxy/script/script.py @@ -4,13 +4,15 @@ Script objects know nothing about mitmproxy or mitmproxy's API - this knowledge by the mitmproxy-specific ScriptContext. """ # Do not import __future__ here, this would apply transitively to the inline scripts. +from __future__ import absolute_import, print_function, division + import os import shlex import sys import six -from ..exceptions import ScriptException +from mitmproxy import exceptions class Script(object): @@ -41,7 +43,7 @@ class Script(object): @staticmethod def parse_command(command): if not command or not command.strip(): - raise ScriptException("Empty script command.") + raise exceptions.ScriptException("Empty script command.") # Windows: escape all backslashes in the path. if os.name == "nt": # pragma: no cover backslashes = shlex.split(command, posix=False)[0].count("\\") @@ -49,13 +51,13 @@ class Script(object): args = shlex.split(command) # pragma: no cover args[0] = os.path.expanduser(args[0]) if not os.path.exists(args[0]): - raise ScriptException( + raise exceptions.ScriptException( ("Script file not found: %s.\r\n" "If your script path contains spaces, " "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % args[0]) elif os.path.isdir(args[0]): - raise ScriptException("Not a file: %s" % args[0]) + raise exceptions.ScriptException("Not a file: %s" % args[0]) return args def load(self): @@ -69,7 +71,7 @@ class Script(object): ScriptException on failure """ if self.ns is not None: - raise ScriptException("Script is already loaded") + raise exceptions.ScriptException("Script is already loaded") script_dir = os.path.dirname(os.path.abspath(self.args[0])) self.ns = {'__file__': os.path.abspath(self.args[0])} sys.path.append(script_dir) @@ -80,8 +82,8 @@ class Script(object): exec(code, self.ns, self.ns) except Exception: six.reraise( - ScriptException, - ScriptException.from_exception_context(), + exceptions.ScriptException, + exceptions.ScriptException.from_exception_context(), sys.exc_info()[2] ) finally: @@ -107,15 +109,15 @@ class Script(object): ScriptException if there was an exception. """ if self.ns is None: - raise ScriptException("Script not loaded.") + raise exceptions.ScriptException("Script not loaded.") f = self.ns.get(name) if f: try: return f(self.ctx, *args, **kwargs) except Exception: six.reraise( - ScriptException, - ScriptException.from_exception_context(), + exceptions.ScriptException, + exceptions.ScriptException.from_exception_context(), sys.exc_info()[2] ) else: diff --git a/mitmproxy/script/script_context.py b/mitmproxy/script/script_context.py index cd5d4b61..44e2736b 100644 --- a/mitmproxy/script/script_context.py +++ b/mitmproxy/script/script_context.py @@ -2,7 +2,8 @@ The mitmproxy script context provides an API to inline scripts. """ from __future__ import absolute_import, print_function, division -from .. import contentviews + +from mitmproxy import contentviews class ScriptContext(object): diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index eb57fa00..6283d845 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,7 +1,8 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function, division import six -from typing import List, Any +from typing import Any +from typing import List import netlib.basetypes diff --git a/mitmproxy/tnetstring.py b/mitmproxy/tnetstring.py index acf563ac..6b1c117a 100644 --- a/mitmproxy/tnetstring.py +++ b/mitmproxy/tnetstring.py @@ -67,9 +67,10 @@ like so:: u'\u03b1' """ -import six from collections import deque +import six + __ver_major__ = 0 __ver_minor__ = 2 __ver_patch__ = 0 diff --git a/mitmproxy/utils.py b/mitmproxy/utils.py index 05342500..672805d0 100644 --- a/mitmproxy/utils.py +++ b/mitmproxy/utils.py @@ -1,7 +1,8 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division + import datetime -import time import json +import time import netlib.utils diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 02b0d900..0ebb0829 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division) +from __future__ import absolute_import, print_function, division from netlib.version import VERSION, IVERSION diff --git a/mitmproxy/web/__init__.py b/mitmproxy/web/__init__.py index 79c81687..80a65886 100644 --- a/mitmproxy/web/__init__.py +++ b/mitmproxy/web/__init__.py @@ -1,14 +1,16 @@ -from __future__ import absolute_import, print_function +from __future__ import absolute_import, print_function, division + import collections -import tornado.ioloop -import tornado.httpserver import sys -from netlib.http import authentication +import tornado.httpserver +import tornado.ioloop -from .. import flow, controller -from ..exceptions import FlowReadException -from . import app +from mitmproxy import controller +from mitmproxy import exceptions +from mitmproxy import flow +from mitmproxy.web import app +from netlib.http import authentication class Stop(Exception): @@ -156,7 +158,7 @@ class WebMaster(flow.FlowMaster): if options.rfile: try: self.load_flows_file(options.rfile) - except FlowReadException as v: + except exceptions.FlowReadException as v: self.add_event( "Could not read flow file: %s" % v, "error" diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index e25c9086..43b2bad1 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -1,14 +1,16 @@ +from __future__ import absolute_import, print_function, division + +import base64 +import json +import logging import os.path import re import six -import tornado.web import tornado.websocket -import logging -import json -import base64 -from .. import version, filt +from mitmproxy import filt +from mitmproxy import version def _strip_content(flow_state): diff --git a/netlib/http/http2/__init__.py b/netlib/http/http2/__init__.py index 7043d36f..633e6a20 100644 --- a/netlib/http/http2/__init__.py +++ b/netlib/http/http2/__init__.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division from .connections import HTTP2Protocol +from netlib.http.http2 import framereader __all__ = [ - "HTTP2Protocol" + "HTTP2Protocol", + "framereader", ] |