diff options
78 files changed, 659 insertions, 2078 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index dae12978..13782ee8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,10 +5,10 @@ environment: CI_DEPS: codecov>=2.0.5 CI_COMMANDS: codecov matrix: - - PYTHON: "C:\\Python27" - TOXENV: "py27" - PYTHON: "C:\\Python35" TOXENV: "py35" + - PYTHON: "C:\\Python27" + TOXENV: "py27" SNAPSHOT_HOST: secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= @@ -35,8 +35,8 @@ deploy_script: ) { pip install -U virtualenv .\dev.ps1 - cmd /c "python .\release\rtool.py bdist 2>&1" - python .\release\rtool.py upload-snapshot --bdist + cmd /c "python -u .\release\rtool.py bdist 2>&1" + python -u .\release\rtool.py upload-snapshot --bdist --wheel } cache: diff --git a/.travis.yml b/.travis.yml index e9566ebe..a8301ec8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,9 @@ matrix: - os: osx osx_image: xcode7.3 language: generic - env: TOXENV=py35 + env: TOXENV=py35 BDIST=1 - python: 3.5 - env: TOXENV=py35 + env: TOXENV=py35 BDIST=1 - python: 3.5 env: TOXENV=py35 NO_ALPN=1 - python: 2.7 @@ -57,14 +57,14 @@ script: set -o pipefail; python -m tox -- --cov netlib --cov mitmproxy --cov pat after_success: - | - if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then git fetch --unshallow ./dev.sh 3.5 source venv3.5/bin/activate pip install -e ./release - python ./release/rtool.py bdist - python ./release/rtool.py upload-snapshot --bdist --wheel + python -u ./release/rtool.py bdist + python -u ./release/rtool.py upload-snapshot --bdist fi notifications: diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 9609a421..a08e2eab 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,10 +1,13 @@ - 1813 Aldo Cortesi - 1228 Maximilian Hils - 282 Thomas Kriechbaumer + 2118 Aldo Cortesi + 1666 Maximilian Hils + 450 Thomas Kriechbaumer + 210 Shadab Zafar + 94 Jason 83 Marcelo Glezer + 36 Clemens 28 Jim Shaver 18 Henrik Nordstrom - 17 Shadab Zafar + 16 Matthew Shao 14 David Weinstein 14 Pedro Worcel 13 Thomas Roth @@ -14,39 +17,47 @@ 10 András Veres-Szentkirályi 10 Chris Czub 10 Sandor Nemes + 10 Zohar Lorberbaum 9 Kyle Morton 9 Legend Tang - 9 Matthew Shao 9 Rouli + 9 ikoz 8 Chandler Abraham 8 Jason A. Novak 7 Alexis Hildebrandt 7 Brad Peabody 7 Matthias Urlichs + 7 dufferzafar + 6 Felix Yan 5 Choongwoo Han 5 Sam Cleveland 5 Tomaz Muraus + 5 Will Coster 5 elitest 5 iroiro123 4 Bryan Bishop + 4 Clemens Brunner 4 Marc Liyanage 4 Michael J. Bazzinotti 4 Valtteri Virtanen 4 Wade 524 4 Youhei Sakurai 4 root + 4 yonder 3 Benjamin Lee 3 Chris Neasbitt 3 Eli Shvartsman - 3 Felix Yan 3 Guillem Anguera 3 Kyle Manna 3 MatthewShao 3 Ryan Welton 3 Zack B + 3 redfast00 + 3 requires.io 2 Anant 2 Bennett Blodinger 2 Colin Bendell + 2 Cory Benfield 2 Heikki Hannikainen 2 Israel Nir 2 Jaime Soriano Pastor @@ -59,34 +70,50 @@ 2 Paul 2 Rob Wills 2 Sean Coates + 2 Steven Van Acker 2 Terry Long 2 Wade Catron 2 alts 2 isra17 2 israel - 2 requires.io + 2 jpkrause + 2 lilydjwg + 2 strohu + 2 依云 + 1 Aditya 1 Andrey Plotnikov 1 Andy Smith + 1 Anthony Zhang + 1 BSalita 1 Ben Lerner 1 Bradley Baetz + 1 Brett Randall 1 Chris Hamant + 1 Christian Frichot 1 Dan Wilbraham 1 David Dworken 1 David Shaw + 1 Doug Freed 1 Doug Lethin + 1 Drake Caraker 1 Eric Entzel 1 Felix Wolfsteller 1 FreeArtMan 1 Gabriel Kirkpatrick 1 Henrik Nordström + 1 Israel Blancas 1 Ivaylo Popov 1 JC 1 Jakub Nawalaniec 1 Jakub Wilk 1 James Billingham + 1 Jason Pepas 1 Jean Regisser + 1 Jonathan Jones 1 Jorge Villacorta 1 Kit Randel + 1 Kostya Esmukov + 1 Linmiao Xu 1 Lucas Cimon 1 M. Utku Altinkaya 1 Mathieu Mitchell @@ -98,27 +125,33 @@ 1 Nick Raptis 1 Nicolas Esteves 1 Oleksandr Sheremet + 1 Parth Ganatra 1 Pritam Baral 1 Rich Somerfield 1 Rory McCann 1 Rune Halvorsen 1 Ryo Onodera + 1 Sachin Kelkar 1 Sahn Lam 1 Seppo Yli-Olli 1 Sergey Chipiga 1 Stefan Wärting 1 Steve Phillips - 1 Steven Van Acker + 1 Steven Noble 1 Suyash + 1 Tai Dickerson 1 Tarashish Mishra 1 TearsDontFalls + 1 Thiago Arrais 1 Tim Becker 1 Timothy Elliott 1 Ulrich Petri 1 Vyacheslav Bakhmutov - 1 Will Coster + 1 Wes Turner + 1 Yoginski 1 Yuangxuan Wang 1 capt8bit + 1 cle1000 1 davidpshaw 1 deployable 1 gecko655 @@ -133,4 +166,3 @@ 1 sethp-jive 1 starenka 1 vzvu3k6k - 1 依云 diff --git a/docs/install.rst b/docs/install.rst index 6d82f81f..6077c3fe 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -13,7 +13,7 @@ This was tested on a fully patched installation of Ubuntu 14.04. .. code:: bash - sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev + sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev g++ sudo pip install mitmproxy # or pip install --user mitmproxy Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. diff --git a/mitmproxy/addons.py b/mitmproxy/addons.py index a4bea9fa..329d1215 100644 --- a/mitmproxy/addons.py +++ b/mitmproxy/addons.py @@ -20,7 +20,7 @@ class Addons(object): def add(self, options, *addons): if not addons: - raise ValueError("No adons specified.") + raise ValueError("No addons specified.") self.chain.extend(addons) for i in addons: self.invoke_with_context(i, "start") diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py index a98c9b46..743ca72e 100644 --- a/mitmproxy/builtins/dumper.py +++ b/mitmproxy/builtins/dumper.py @@ -1,7 +1,6 @@ from __future__ import absolute_import, print_function, division import itertools -import traceback import click @@ -243,4 +242,4 @@ class Dumper(object): server=repr(f.server_conn.address), direction=direction, )) - self._echo_message(message)
\ No newline at end of file + self._echo_message(message) diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index c960dd1c..ae1d1b91 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -61,13 +61,13 @@ def scriptenv(path, args): try: yield except Exception: - _, _, tb = sys.exc_info() + etype, value, tb = sys.exc_info() scriptdir = os.path.dirname(os.path.abspath(path)) for i, s in enumerate(reversed(traceback.extract_tb(tb))): tb = tb.tb_next if not os.path.abspath(s[0]).startswith(scriptdir): break - ctx.log.error("Script error: %s" % "".join(traceback.format_tb(tb))) + ctx.log.error("Script error: %s" % "".join(traceback.format_exception(etype, value, tb))) finally: sys.argv = oldargs sys.path.pop() diff --git a/mitmproxy/cmdline.py b/mitmproxy/cmdline.py index a6844241..d888b93f 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/cmdline.py @@ -260,7 +260,7 @@ def get_common_options(args): upstream_auth = args.upstream_auth, ssl_version_client = args.ssl_version_client, ssl_version_server = args.ssl_version_server, - ssl_verify_upstream_cert = args.ssl_verify_upstream_cert, + ssl_insecure = args.ssl_insecure, ssl_verify_upstream_trusted_cadir = args.ssl_verify_upstream_trusted_cadir, ssl_verify_upstream_trusted_ca = args.ssl_verify_upstream_trusted_ca, tcp_hosts = args.tcp_hosts, @@ -519,10 +519,9 @@ def proxy_ssl_options(parser): "that will be served to the proxy client, as extras." ) group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." + "--insecure", default=False, + action="store_true", dest="ssl_insecure", + help="Do not verify upstream server SSL/TLS certificates." ) group.add_argument( "--upstream-trusted-cadir", default=None, action="store", @@ -773,7 +772,7 @@ def mitmproxy(): help="Show event log." ) parser.add_argument( - "-f", "--follow", + "--follow", action="store_true", dest="follow", help="Follow flow list." ) @@ -792,9 +791,9 @@ def mitmproxy(): help="Intercept filter expression." ) group.add_argument( - "-l", "--limit", action="store", - type=str, dest="limit", default=None, - help="Limit filter expression." + "-f", "--filter", action="store", + type=str, dest="filter", default=None, + help="Filter view expression." ) return parser diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index 9fb8b5c9..2eb6a7d9 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -379,7 +379,7 @@ def raw_format_flow(f, focus, extended): 4: "code_400", 5: "code_500", } - ccol = codes.get(f["resp_code"] / 100, "code_other") + ccol = codes.get(f["resp_code"] // 100, "code_other") resp.append(fcol(SYMBOL_RETURN, ccol)) if f["resp_is_replay"]: resp.append(fcol(SYMBOL_REPLAY, "replay")) diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index 43742083..12caf315 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -18,8 +18,8 @@ def _mkhelp(): ("d", "delete flow"), ("D", "duplicate flow"), ("e", "toggle eventlog"), + ("f", "filter view"), ("F", "toggle follow flow list"), - ("l", "set limit filter pattern"), ("L", "load saved flows"), ("m", "toggle flow mark"), ("M", "toggle marked flow view"), @@ -367,11 +367,11 @@ class FlowListBox(urwid.ListBox): elif key == "G": self.master.state.set_focus(self.master.state.flow_count()) signals.flowlist_change.send(self) - elif key == "l": + elif key == "f": signals.status_prompt.send( - prompt = "Limit", - text = self.master.state.limit_txt, - callback = self.master.set_limit + prompt = "Filter View", + text = self.master.state.filter_txt, + callback = self.master.set_view_filter ) elif key == "L": signals.status_prompt_path.send( diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index 323aefd2..1c3c4e98 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -3,14 +3,12 @@ from __future__ import absolute_import, print_function, division import math import os import sys -import traceback import urwid from typing import Optional, Union # noqa from mitmproxy import contentviews from mitmproxy import controller -from mitmproxy import exceptions from mitmproxy import models from mitmproxy import utils from mitmproxy.console import common @@ -705,4 +703,4 @@ class FlowView(tabs.Tabs): "b": "brotli", } conn.encode(encoding_map[key]) - signals.flow_change.send(self, flow = self.flow)
\ No newline at end of file + signals.flow_change.send(self, flow = self.flow) diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 03ec8b63..18a4c1f0 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -75,8 +75,8 @@ class ConsoleState(flow.State): self.update_focus() return f - def set_limit(self, limit): - ret = super(ConsoleState, self).set_limit(limit) + def set_view_filter(self, txt): + ret = super(ConsoleState, self).set_view_filter(txt) self.set_focus(self.focus) return ret @@ -153,8 +153,8 @@ class ConsoleState(flow.State): last_focus, _ = self.get_focus() nearest_marked = self.get_nearest_matching_flow(last_focus, marked_filter) - self.last_filter = self.limit_txt - self.set_limit(marked_filter) + self.last_filter = self.filter_txt + self.set_view_filter(marked_filter) # Restore Focus if last_focus.marked: @@ -171,7 +171,7 @@ class ConsoleState(flow.State): last_focus, _ = self.get_focus() nearest_marked = self.get_nearest_matching_flow(last_focus, marked_filter) - self.set_limit(self.last_filter) + self.set_view_filter(self.last_filter) self.last_filter = "" # Restore Focus @@ -203,7 +203,7 @@ class Options(mitmproxy.options.Options): eventlog=False, # type: bool follow=False, # type: bool intercept=False, # type: bool - limit=None, # type: Optional[str] + filter=None, # type: Optional[str] palette=None, # type: Optional[str] palette_transparent=False, # type: bool no_mouse=False, # type: bool @@ -212,7 +212,7 @@ class Options(mitmproxy.options.Options): self.eventlog = eventlog self.follow = follow self.intercept = intercept - self.limit = limit + self.filter = filter self.palette = palette self.palette_transparent = palette_transparent self.no_mouse = no_mouse @@ -234,8 +234,8 @@ class ConsoleMaster(flow.FlowMaster): print("Intercept error: {}".format(r), file=sys.stderr) sys.exit(1) - if options.limit: - self.set_limit(options.limit) + if options.filter: + self.set_view_filter(options.filter) self.set_stream_large_bodies(options.stream_large_bodies) @@ -258,6 +258,7 @@ class ConsoleMaster(flow.FlowMaster): signals.call_in.connect(self.sig_call_in) signals.pop_view_state.connect(self.sig_pop_view_state) + signals.replace_view_state.connect(self.sig_replace_view_state) signals.push_view_state.connect(self.sig_push_view_state) signals.sig_add_log.connect(self.sig_add_log) self.addons.add(options, *builtins.default_addons()) @@ -276,11 +277,11 @@ class ConsoleMaster(flow.FlowMaster): if self.options.verbosity < utils.log_tier(level): return - if level == "error": + if level in ("error", "warn"): signals.status_message.send( - message = "Error: %s" % str(e) + message = "{}: {}".format(level.title(), e) ) - e = urwid.Text(("error", str(e))) + e = urwid.Text((level, str(e))) else: e = urwid.Text(str(e)) self.logbuffer.append(e) @@ -296,7 +297,19 @@ class ConsoleMaster(flow.FlowMaster): return callback(*args) self.loop.set_alarm_in(seconds, cb) + def sig_replace_view_state(self, sender): + """ + A view has been pushed onto the stack, and is intended to replace + the current view rather tha creating a new stack entry. + """ + if len(self.view_stack) > 1: + del self.view_stack[1] + def sig_pop_view_state(self, sender): + """ + Pop the top view off the view stack. If no more views will be left + after this, prompt for exit. + """ if len(self.view_stack) > 1: self.view_stack.pop() self.loop.widget = self.view_stack[-1] @@ -312,6 +325,9 @@ class ConsoleMaster(flow.FlowMaster): ) def sig_push_view_state(self, sender, window): + """ + Push a new view onto the view stack. + """ self.view_stack.append(window) self.loop.widget = window self.loop.draw_screen() @@ -352,8 +368,8 @@ class ConsoleMaster(flow.FlowMaster): def toggle_eventlog(self): self.options.eventlog = not self.options.eventlog - signals.pop_view_state.send(self) self.view_flowlist() + signals.replace_view_state.send(self) def _readflows(self, path): """ @@ -656,8 +672,8 @@ class ConsoleMaster(flow.FlowMaster): def accept_all(self): self.state.accept_all(self) - def set_limit(self, txt): - v = self.state.set_limit(txt) + def set_view_filter(self, txt): + v = self.state.set_view_filter(txt) signals.flowlist_change.send(self) return v diff --git a/mitmproxy/console/options.py b/mitmproxy/console/options.py index f9fc3764..f7fb2f90 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/console/options.py @@ -91,6 +91,12 @@ class Options(urwid.WidgetWrap): lambda: master.options.tcp_hosts, self.tcp_hosts ), + select.Option( + "Don't Verify SSL/TLS Certificates", + "V", + lambda: master.options.ssl_insecure, + master.options.toggler("ssl_insecure") + ), select.Heading("Utility"), select.Option( @@ -134,15 +140,17 @@ class Options(urwid.WidgetWrap): title = urwid.Text("Options") title = urwid.Padding(title, align="left", width=("relative", 100)) title = urwid.AttrWrap(title, "heading") - self._w = urwid.Frame( + w = urwid.Frame( self.lb, header = title ) + super(Options, self).__init__(w) + self.master.loop.widget.footer.update("") signals.update_settings.connect(self.sig_update_settings) - master.options.changed.connect(lambda sender, updated: self.sig_update_settings(sender)) + master.options.changed.connect(self.sig_update_settings) - def sig_update_settings(self, sender): + def sig_update_settings(self, sender, updated=None): self.lb.walker._modified() def keypress(self, size, key): diff --git a/mitmproxy/console/signals.py b/mitmproxy/console/signals.py index 97507834..93eb399f 100644 --- a/mitmproxy/console/signals.py +++ b/mitmproxy/console/signals.py @@ -43,3 +43,4 @@ flowlist_change = blinker.Signal() # Pop and push view state onto a stack pop_view_state = blinker.Signal() push_view_state = blinker.Signal() +replace_view_state = blinker.Signal() diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/console/statusbar.py index 156d1176..43d68d51 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/console/statusbar.py @@ -124,10 +124,10 @@ class StatusBar(urwid.WidgetWrap): super(StatusBar, self).__init__(urwid.Pile([self.ib, self.master.ab])) signals.update_settings.connect(self.sig_update_settings) signals.flowlist_change.connect(self.sig_update_settings) - master.options.changed.connect(lambda sender, updated: self.sig_update_settings(sender)) + master.options.changed.connect(self.sig_update_settings) self.redraw() - def sig_update_settings(self, sender): + def sig_update_settings(self, sender, updated=None): self.redraw() def keypress(self, *args, **kwargs): @@ -167,10 +167,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "i")) r.append(":%s]" % self.master.state.intercept_txt) - if self.master.state.limit_txt: + if self.master.state.filter_txt: r.append("[") - r.append(("heading_key", "l")) - r.append(":%s]" % self.master.state.limit_txt) + r.append(("heading_key", "f")) + r.append(":%s]" % self.master.state.filter_txt) if self.master.options.stickycookie: r.append("[") r.append(("heading_key", "t")) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 807638dc..f95efb99 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -668,4 +668,4 @@ def get_content_view(viewmode, data, **metadata): traceback.format_exc() ) - return desc, safe_to_print(content), error
\ No newline at end of file + return desc, safe_to_print(content), error diff --git a/mitmproxy/contrib/jsbeautifier/__init__.py b/mitmproxy/contrib/jsbeautifier/__init__.py deleted file mode 100644 index e319e8dd..00000000 --- a/mitmproxy/contrib/jsbeautifier/__init__.py +++ /dev/null @@ -1,1153 +0,0 @@ -import sys -import getopt -import re -import string - -# -# Originally written by Einar Lielmanis et al., -# Conversion to python by Einar Lielmanis, einar@jsbeautifier.org, -# MIT licence, enjoy. -# -# Python is not my native language, feel free to push things around. -# -# Use either from command line (script displays its usage when run -# without any parameters), -# -# -# or, alternatively, use it as a module: -# -# import jsbeautifier -# res = jsbeautifier.beautify('your javascript string') -# res = jsbeautifier.beautify_file('some_file.js') -# -# you may specify some options: -# -# opts = jsbeautifier.default_options() -# opts.indent_size = 2 -# res = jsbeautifier.beautify('some javascript', opts) -# -# -# Here are the available options: (read source) - - -class BeautifierOptions: - def __init__(self): - self.indent_size = 4 - self.indent_char = ' ' - self.indent_with_tabs = False - self.preserve_newlines = True - self.max_preserve_newlines = 10. - self.jslint_happy = False - self.brace_style = 'collapse' - self.keep_array_indentation = False - self.keep_function_indentation = False - self.eval_code = False - - - - def __repr__(self): - return \ -"""indent_size = %d -indent_char = [%s] -preserve_newlines = %s -max_preserve_newlines = %d -jslint_happy = %s -indent_with_tabs = %s -brace_style = %s -keep_array_indentation = %s -eval_code = %s -""" % ( self.indent_size, - self.indent_char, - self.preserve_newlines, - self.max_preserve_newlines, - self.jslint_happy, - self.indent_with_tabs, - self.brace_style, - self.keep_array_indentation, - self.eval_code, - ) - - -class BeautifierFlags: - def __init__(self, mode): - self.previous_mode = 'BLOCK' - self.mode = mode - self.var_line = False - self.var_line_tainted = False - self.var_line_reindented = False - self.in_html_comment = False - self.if_line = False - self.in_case = False - self.eat_next_space = False - self.indentation_baseline = -1 - self.indentation_level = 0 - self.ternary_depth = 0 - - -def default_options(): - return BeautifierOptions() - - -def beautify(string, opts = default_options() ): - b = Beautifier() - return b.beautify(string, opts) - -def beautify_file(file_name, opts = default_options() ): - - if file_name == '-': # stdin - f = sys.stdin - else: - try: - f = open(file_name) - except Exception as ex: - return 'The file could not be opened' - - b = Beautifier() - return b.beautify(''.join(f.readlines()), opts) - - -def usage(): - - print("""Javascript beautifier (http://jsbeautifier.org/) - -Usage: jsbeautifier.py [options] <infile> - - <infile> can be "-", which means stdin. - <outfile> defaults to stdout - -Input options: - - -i, --stdin read input from stdin - -Output options: - - -s, --indent-size=NUMBER indentation size. (default 4). - -c, --indent-char=CHAR character to indent with. (default space). - -t, --indent-with-tabs Indent with tabs, overrides -s and -c - -d, --disable-preserve-newlines do not preserve existing line breaks. - -j, --jslint-happy more jslint-compatible output - -b, --brace-style=collapse brace style (collapse, expand, end-expand) - -k, --keep-array-indentation keep array indentation. - -o, --outfile=FILE specify a file to output to (default stdout) - -f, --keep-function-indentation Do not re-indent function bodies defined in var lines. - -Rarely needed options: - - --eval-code evaluate code if a JS interpreter is - installed. May be useful with some obfuscated - script but poses a potential security issue. - - -l, --indent-level=NUMBER initial indentation level. (default 0). - - -h, --help, --usage prints this help statement. - -""") - - - - - - -class Beautifier: - - def __init__(self, opts = default_options() ): - - self.opts = opts - self.blank_state() - - def blank_state(self): - - # internal flags - self.flags = BeautifierFlags('BLOCK') - self.flag_store = [] - self.wanted_newline = False - self.just_added_newline = False - self.do_block_just_closed = False - - if self.opts.indent_with_tabs: - self.indent_string = "\t" - else: - self.indent_string = self.opts.indent_char * self.opts.indent_size - - self.preindent_string = '' - self.last_word = '' # last TK_WORD seen - self.last_type = 'TK_START_EXPR' # last token type - self.last_text = '' # last token text - self.last_last_text = '' # pre-last token text - - self.input = None - self.output = [] # formatted javascript gets built here - - self.whitespace = ["\n", "\r", "\t", " "] - self.wordchar = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$' - self.digits = '0123456789' - self.punct = '+ - * / % & ++ -- = += -= *= /= %= == === != !== > < >= <= >> << >>> >>>= >>= <<= && &= | || ! !! , : ? ^ ^= |= ::' - self.punct += ' <?= <? ?> <%= <% %>' - self.punct = self.punct.split(' ') - - - # Words which always should start on a new line - self.line_starters = 'continue,try,throw,return,var,if,switch,case,default,for,while,break,function'.split(',') - self.set_mode('BLOCK') - - global parser_pos - parser_pos = 0 - - - def beautify(self, s, opts = None ): - - if opts != None: - self.opts = opts - - - if self.opts.brace_style not in ['expand', 'collapse', 'end-expand']: - raise(Exception('opts.brace_style must be "expand", "collapse" or "end-expand".')) - - self.blank_state() - - while s and s[0] in [' ', '\t']: - self.preindent_string += s[0] - s = s[1:] - - #self.input = self.unpack(s, opts.eval_code) - # CORTESI - self.input = s - - parser_pos = 0 - while True: - token_text, token_type = self.get_next_token() - #print (token_text, token_type, self.flags.mode) - if token_type == 'TK_EOF': - break - - handlers = { - 'TK_START_EXPR': self.handle_start_expr, - 'TK_END_EXPR': self.handle_end_expr, - 'TK_START_BLOCK': self.handle_start_block, - 'TK_END_BLOCK': self.handle_end_block, - 'TK_WORD': self.handle_word, - 'TK_SEMICOLON': self.handle_semicolon, - 'TK_STRING': self.handle_string, - 'TK_EQUALS': self.handle_equals, - 'TK_OPERATOR': self.handle_operator, - 'TK_BLOCK_COMMENT': self.handle_block_comment, - 'TK_INLINE_COMMENT': self.handle_inline_comment, - 'TK_COMMENT': self.handle_comment, - 'TK_UNKNOWN': self.handle_unknown, - } - - handlers[token_type](token_text) - - self.last_last_text = self.last_text - self.last_type = token_type - self.last_text = token_text - - sweet_code = self.preindent_string + re.sub('[\n ]+$', '', ''.join(self.output)) - return sweet_code - - def unpack(self, source, evalcode=False): - import jsbeautifier.unpackers as unpackers - try: - return unpackers.run(source, evalcode) - except unpackers.UnpackingError as error: - print('error:', error) - return '' - - def trim_output(self, eat_newlines = False): - while len(self.output) \ - and ( - self.output[-1] == ' '\ - or self.output[-1] == self.indent_string \ - or self.output[-1] == self.preindent_string \ - or (eat_newlines and self.output[-1] in ['\n', '\r'])): - self.output.pop() - - def is_special_word(self, s): - return s in ['case', 'return', 'do', 'if', 'throw', 'else']; - - def is_array(self, mode): - return mode in ['[EXPRESSION]', '[INDENDED-EXPRESSION]'] - - - def is_expression(self, mode): - return mode in ['[EXPRESSION]', '[INDENDED-EXPRESSION]', '(EXPRESSION)', '(FOR-EXPRESSION)', '(COND-EXPRESSION)'] - - - def append_newline_forced(self): - old_array_indentation = self.opts.keep_array_indentation - self.opts.keep_array_indentation = False - self.append_newline() - self.opts.keep_array_indentation = old_array_indentation - - def append_newline(self, ignore_repeated = True): - - self.flags.eat_next_space = False - - if self.opts.keep_array_indentation and self.is_array(self.flags.mode): - return - - self.flags.if_line = False - self.trim_output() - - if len(self.output) == 0: - # no newline on start of file - return - - if self.output[-1] != '\n' or not ignore_repeated: - self.just_added_newline = True - self.output.append('\n') - - if self.preindent_string: - self.output.append(self.preindent_string) - - for i in range(self.flags.indentation_level): - self.output.append(self.indent_string) - - if self.flags.var_line and self.flags.var_line_reindented: - self.output.append(self.indent_string) - - - def append(self, s): - if s == ' ': - # do not add just a single space after the // comment, ever - if self.last_type == 'TK_COMMENT': - return self.append_newline() - - # make sure only single space gets drawn - if self.flags.eat_next_space: - self.flags.eat_next_space = False - elif len(self.output) and self.output[-1] not in [' ', '\n', self.indent_string]: - self.output.append(' ') - else: - self.just_added_newline = False - self.flags.eat_next_space = False - self.output.append(s) - - - def indent(self): - self.flags.indentation_level = self.flags.indentation_level + 1 - - - def remove_indent(self): - if len(self.output) and self.output[-1] in [self.indent_string, self.preindent_string]: - self.output.pop() - - - def set_mode(self, mode): - - prev = BeautifierFlags('BLOCK') - - if self.flags: - self.flag_store.append(self.flags) - prev = self.flags - - self.flags = BeautifierFlags(mode) - - if len(self.flag_store) == 1: - self.flags.indentation_level = 0 - else: - self.flags.indentation_level = prev.indentation_level - if prev.var_line and prev.var_line_reindented: - self.flags.indentation_level = self.flags.indentation_level + 1 - self.flags.previous_mode = prev.mode - - - def restore_mode(self): - self.do_block_just_closed = self.flags.mode == 'DO_BLOCK' - if len(self.flag_store) > 0: - mode = self.flags.mode - self.flags = self.flag_store.pop() - self.flags.previous_mode = mode - - - def get_next_token(self): - - global parser_pos - - self.n_newlines = 0 - - if parser_pos >= len(self.input): - return '', 'TK_EOF' - - self.wanted_newline = False - c = self.input[parser_pos] - parser_pos += 1 - - keep_whitespace = self.opts.keep_array_indentation and self.is_array(self.flags.mode) - - if keep_whitespace: - # slight mess to allow nice preservation of array indentation and reindent that correctly - # first time when we get to the arrays: - # var a = [ - # ....'something' - # we make note of whitespace_count = 4 into flags.indentation_baseline - # so we know that 4 whitespaces in original source match indent_level of reindented source - # - # and afterwards, when we get to - # 'something, - # .......'something else' - # we know that this should be indented to indent_level + (7 - indentation_baseline) spaces - - whitespace_count = 0 - while c in self.whitespace: - if c == '\n': - self.trim_output() - self.output.append('\n') - self.just_added_newline = True - whitespace_count = 0 - elif c == '\t': - whitespace_count += 4 - elif c == '\r': - pass - else: - whitespace_count += 1 - - if parser_pos >= len(self.input): - return '', 'TK_EOF' - - c = self.input[parser_pos] - parser_pos += 1 - - if self.flags.indentation_baseline == -1: - - self.flags.indentation_baseline = whitespace_count - - if self.just_added_newline: - for i in range(self.flags.indentation_level + 1): - self.output.append(self.indent_string) - - if self.flags.indentation_baseline != -1: - for i in range(whitespace_count - self.flags.indentation_baseline): - self.output.append(' ') - - else: # not keep_whitespace - while c in self.whitespace: - if c == '\n': - if self.opts.max_preserve_newlines == 0 or self.opts.max_preserve_newlines > self.n_newlines: - self.n_newlines += 1 - - if parser_pos >= len(self.input): - return '', 'TK_EOF' - - c = self.input[parser_pos] - parser_pos += 1 - - if self.opts.preserve_newlines and self.n_newlines > 1: - for i in range(self.n_newlines): - self.append_newline(i == 0) - self.just_added_newline = True - - self.wanted_newline = self.n_newlines > 0 - - - if c in self.wordchar: - if parser_pos < len(self.input): - while self.input[parser_pos] in self.wordchar: - c = c + self.input[parser_pos] - parser_pos += 1 - if parser_pos == len(self.input): - break - - # small and surprisingly unugly hack for 1E-10 representation - if parser_pos != len(self.input) and self.input[parser_pos] in '+-' \ - and re.match('^[0-9]+[Ee]$', c): - - sign = self.input[parser_pos] - parser_pos += 1 - t = self.get_next_token() - c += sign + t[0] - return c, 'TK_WORD' - - if c == 'in': # in is an operator, need to hack - return c, 'TK_OPERATOR' - - if self.wanted_newline and \ - self.last_type != 'TK_OPERATOR' and\ - self.last_type != 'TK_EQUALS' and\ - not self.flags.if_line and \ - (self.opts.preserve_newlines or self.last_text != 'var'): - self.append_newline() - - return c, 'TK_WORD' - - if c in '([': - return c, 'TK_START_EXPR' - - if c in ')]': - return c, 'TK_END_EXPR' - - if c == '{': - return c, 'TK_START_BLOCK' - - if c == '}': - return c, 'TK_END_BLOCK' - - if c == ';': - return c, 'TK_SEMICOLON' - - if c == '/': - comment = '' - inline_comment = True - comment_mode = 'TK_INLINE_COMMENT' - if self.input[parser_pos] == '*': # peek /* .. */ comment - parser_pos += 1 - if parser_pos < len(self.input): - while not (self.input[parser_pos] == '*' and \ - parser_pos + 1 < len(self.input) and \ - self.input[parser_pos + 1] == '/')\ - and parser_pos < len(self.input): - c = self.input[parser_pos] - comment += c - if c in '\r\n': - comment_mode = 'TK_BLOCK_COMMENT' - parser_pos += 1 - if parser_pos >= len(self.input): - break - parser_pos += 2 - return '/*' + comment + '*/', comment_mode - if self.input[parser_pos] == '/': # peek // comment - comment = c - while self.input[parser_pos] not in '\r\n': - comment += self.input[parser_pos] - parser_pos += 1 - if parser_pos >= len(self.input): - break - parser_pos += 1 - if self.wanted_newline: - self.append_newline() - return comment, 'TK_COMMENT' - - - - if c == "'" or c == '"' or \ - (c == '/' and ((self.last_type == 'TK_WORD' and self.is_special_word(self.last_text)) or \ - (self.last_type == 'TK_END_EXPR' and self.flags.previous_mode in ['(FOR-EXPRESSION)', '(COND-EXPRESSION)']) or \ - (self.last_type in ['TK_COMMENT', 'TK_START_EXPR', 'TK_START_BLOCK', 'TK_END_BLOCK', 'TK_OPERATOR', - 'TK_EQUALS', 'TK_EOF', 'TK_SEMICOLON']))): - sep = c - esc = False - resulting_string = c - in_char_class = False - - if parser_pos < len(self.input): - if sep == '/': - # handle regexp - in_char_class = False - while esc or in_char_class or self.input[parser_pos] != sep: - resulting_string += self.input[parser_pos] - if not esc: - esc = self.input[parser_pos] == '\\' - if self.input[parser_pos] == '[': - in_char_class = True - elif self.input[parser_pos] == ']': - in_char_class = False - else: - esc = False - parser_pos += 1 - if parser_pos >= len(self.input): - # incomplete regex when end-of-file reached - # bail out with what has received so far - return resulting_string, 'TK_STRING' - else: - # handle string - while esc or self.input[parser_pos] != sep: - resulting_string += self.input[parser_pos] - if not esc: - esc = self.input[parser_pos] == '\\' - else: - esc = False - parser_pos += 1 - if parser_pos >= len(self.input): - # incomplete string when end-of-file reached - # bail out with what has received so far - return resulting_string, 'TK_STRING' - - - parser_pos += 1 - resulting_string += sep - if sep == '/': - # regexps may have modifiers /regexp/MOD, so fetch those too - while parser_pos < len(self.input) and self.input[parser_pos] in self.wordchar: - resulting_string += self.input[parser_pos] - parser_pos += 1 - return resulting_string, 'TK_STRING' - - if c == '#': - - # she-bang - if len(self.output) == 0 and len(self.input) > 1 and self.input[parser_pos] == '!': - resulting_string = c - while parser_pos < len(self.input) and c != '\n': - c = self.input[parser_pos] - resulting_string += c - parser_pos += 1 - self.output.append(resulting_string.strip() + "\n") - self.append_newline() - return self.get_next_token() - - - # Spidermonkey-specific sharp variables for circular references - # https://developer.mozilla.org/En/Sharp_variables_in_JavaScript - # http://mxr.mozilla.org/mozilla-central/source/js/src/jsscan.cpp around line 1935 - sharp = '#' - if parser_pos < len(self.input) and self.input[parser_pos] in self.digits: - while True: - c = self.input[parser_pos] - sharp += c - parser_pos += 1 - if parser_pos >= len(self.input) or c == '#' or c == '=': - break - if c == '#' or parser_pos >= len(self.input): - pass - elif self.input[parser_pos] == '[' and self.input[parser_pos + 1] == ']': - sharp += '[]' - parser_pos += 2 - elif self.input[parser_pos] == '{' and self.input[parser_pos + 1] == '}': - sharp += '{}' - parser_pos += 2 - return sharp, 'TK_WORD' - - if c == '<' and self.input[parser_pos - 1 : parser_pos + 3] == '<!--': - parser_pos += 3 - c = '<!--' - while parser_pos < len(self.input) and self.input[parser_pos] != '\n': - c += self.input[parser_pos] - parser_pos += 1 - self.flags.in_html_comment = True - return c, 'TK_COMMENT' - - if c == '-' and self.flags.in_html_comment and self.input[parser_pos - 1 : parser_pos + 2] == '-->': - self.flags.in_html_comment = False - parser_pos += 2 - if self.wanted_newline: - self.append_newline() - return '-->', 'TK_COMMENT' - - if c in self.punct: - while parser_pos < len(self.input) and c + self.input[parser_pos] in self.punct: - c += self.input[parser_pos] - parser_pos += 1 - if parser_pos >= len(self.input): - break - if c == '=': - return c, 'TK_EQUALS' - else: - return c, 'TK_OPERATOR' - return c, 'TK_UNKNOWN' - - - - def handle_start_expr(self, token_text): - if token_text == '[': - if self.last_type == 'TK_WORD' or self.last_text == ')': - if self.last_text in self.line_starters: - self.append(' ') - self.set_mode('(EXPRESSION)') - self.append(token_text) - return - - if self.flags.mode in ['[EXPRESSION]', '[INDENTED-EXPRESSION]']: - if self.last_last_text == ']' and self.last_text == ',': - # ], [ goes to a new line - if self.flags.mode == '[EXPRESSION]': - self.flags.mode = '[INDENTED-EXPRESSION]' - if not self.opts.keep_array_indentation: - self.indent() - self.set_mode('[EXPRESSION]') - if not self.opts.keep_array_indentation: - self.append_newline() - elif self.last_text == '[': - if self.flags.mode == '[EXPRESSION]': - self.flags.mode = '[INDENTED-EXPRESSION]' - if not self.opts.keep_array_indentation: - self.indent() - self.set_mode('[EXPRESSION]') - - if not self.opts.keep_array_indentation: - self.append_newline() - else: - self.set_mode('[EXPRESSION]') - else: - self.set_mode('[EXPRESSION]') - else: - if self.last_text == 'for': - self.set_mode('(FOR-EXPRESSION)') - elif self.last_text in ['if', 'while']: - self.set_mode('(COND-EXPRESSION)') - else: - self.set_mode('(EXPRESSION)') - - - if self.last_text == ';' or self.last_type == 'TK_START_BLOCK': - self.append_newline() - elif self.last_type in ['TK_END_EXPR', 'TK_START_EXPR', 'TK_END_BLOCK'] or self.last_text == '.': - # do nothing on (( and )( and ][ and ]( and .( - if self.wanted_newline: - self.append_newline(); - elif self.last_type not in ['TK_WORD', 'TK_OPERATOR']: - self.append(' ') - elif self.last_word == 'function' or self.last_word == 'typeof': - # function() vs function (), typeof() vs typeof () - if self.opts.jslint_happy: - self.append(' ') - elif self.last_text in self.line_starters or self.last_text == 'catch': - self.append(' ') - - self.append(token_text) - - - def handle_end_expr(self, token_text): - if token_text == ']': - if self.opts.keep_array_indentation: - if self.last_text == '}': - self.remove_indent() - self.append(token_text) - self.restore_mode() - return - else: - if self.flags.mode == '[INDENTED-EXPRESSION]': - if self.last_text == ']': - self.restore_mode() - self.append_newline() - self.append(token_text) - return - self.restore_mode() - self.append(token_text) - - - def handle_start_block(self, token_text): - if self.last_word == 'do': - self.set_mode('DO_BLOCK') - else: - self.set_mode('BLOCK') - - if self.opts.brace_style == 'expand': - if self.last_type != 'TK_OPERATOR': - if self.last_text == '=' or (self.is_special_word(self.last_text) and self.last_text != 'else'): - self.append(' ') - else: - self.append_newline(True) - - self.append(token_text) - self.indent() - else: - if self.last_type not in ['TK_OPERATOR', 'TK_START_EXPR']: - if self.last_type == 'TK_START_BLOCK': - self.append_newline() - else: - self.append(' ') - else: - # if TK_OPERATOR or TK_START_EXPR - if self.is_array(self.flags.previous_mode) and self.last_text == ',': - if self.last_last_text == '}': - self.append(' ') - else: - self.append_newline() - self.indent() - self.append(token_text) - - - def handle_end_block(self, token_text): - self.restore_mode() - if self.opts.brace_style == 'expand': - if self.last_text != '{': - self.append_newline() - else: - if self.last_type == 'TK_START_BLOCK': - if self.just_added_newline: - self.remove_indent() - else: - # {} - self.trim_output() - else: - if self.is_array(self.flags.mode) and self.opts.keep_array_indentation: - self.opts.keep_array_indentation = False - self.append_newline() - self.opts.keep_array_indentation = True - else: - self.append_newline() - - self.append(token_text) - - - def handle_word(self, token_text): - if self.do_block_just_closed: - self.append(' ') - self.append(token_text) - self.append(' ') - self.do_block_just_closed = False - return - - if token_text == 'function': - - if self.flags.var_line: - self.flags.var_line_reindented = not self.opts.keep_function_indentation - if (self.just_added_newline or self.last_text == ';') and self.last_text != '{': - # make sure there is a nice clean space of at least one blank line - # before a new function definition - have_newlines = self.n_newlines - if not self.just_added_newline: - have_newlines = 0 - if not self.opts.preserve_newlines: - have_newlines = 1 - for i in range(2 - have_newlines): - self.append_newline(False) - - if token_text in ['case', 'default']: - if self.last_text == ':': - self.remove_indent() - else: - self.flags.indentation_level -= 1 - self.append_newline() - self.flags.indentation_level += 1 - self.append(token_text) - self.flags.in_case = True - return - - prefix = 'NONE' - - if self.last_type == 'TK_END_BLOCK': - if token_text not in ['else', 'catch', 'finally']: - prefix = 'NEWLINE' - else: - if self.opts.brace_style in ['expand', 'end-expand']: - prefix = 'NEWLINE' - else: - prefix = 'SPACE' - self.append(' ') - elif self.last_type == 'TK_SEMICOLON' and self.flags.mode in ['BLOCK', 'DO_BLOCK']: - prefix = 'NEWLINE' - elif self.last_type == 'TK_SEMICOLON' and self.is_expression(self.flags.mode): - prefix = 'SPACE' - elif self.last_type == 'TK_STRING': - prefix = 'NEWLINE' - elif self.last_type == 'TK_WORD': - if self.last_text == 'else': - # eat newlines between ...else *** some_op... - # won't preserve extra newlines in this place (if any), but don't care that much - self.trim_output(True) - prefix = 'SPACE' - elif self.last_type == 'TK_START_BLOCK': - prefix = 'NEWLINE' - elif self.last_type == 'TK_END_EXPR': - self.append(' ') - prefix = 'NEWLINE' - - if self.flags.if_line and self.last_type == 'TK_END_EXPR': - self.flags.if_line = False - - if token_text in self.line_starters: - if self.last_text == 'else': - prefix = 'SPACE' - else: - prefix = 'NEWLINE' - - if token_text == 'function' and self.last_text in ['get', 'set']: - prefix = 'SPACE' - - if token_text in ['else', 'catch', 'finally']: - if self.last_type != 'TK_END_BLOCK' \ - or self.opts.brace_style == 'expand' \ - or self.opts.brace_style == 'end-expand': - self.append_newline() - else: - self.trim_output(True) - self.append(' ') - elif prefix == 'NEWLINE': - if token_text == 'function' and (self.last_type == 'TK_START_EXPR' or self.last_text in '=,'): - # no need to force newline on "function" - - # (function... - pass - elif token_text == 'function' and self.last_text == 'new': - self.append(' ') - elif self.is_special_word(self.last_text): - # no newline between return nnn - self.append(' ') - elif self.last_type != 'TK_END_EXPR': - if (self.last_type != 'TK_START_EXPR' or token_text != 'var') and self.last_text != ':': - # no need to force newline on VAR - - # for (var x = 0... - if token_text == 'if' and self.last_word == 'else' and self.last_text != '{': - self.append(' ') - else: - self.flags.var_line = False - self.flags.var_line_reindented = False - self.append_newline() - elif token_text in self.line_starters and self.last_text != ')': - self.flags.var_line = False - self.flags.var_line_reindented = False - self.append_newline() - elif self.is_array(self.flags.mode) and self.last_text == ',' and self.last_last_text == '}': - self.append_newline() # }, in lists get a newline - elif prefix == 'SPACE': - self.append(' ') - - - self.append(token_text) - self.last_word = token_text - - if token_text == 'var': - self.flags.var_line = True - self.flags.var_line_reindented = False - self.flags.var_line_tainted = False - - - if token_text == 'if': - self.flags.if_line = True - - if token_text == 'else': - self.flags.if_line = False - - - def handle_semicolon(self, token_text): - self.append(token_text) - self.flags.var_line = False - self.flags.var_line_reindented = False - if self.flags.mode == 'OBJECT': - # OBJECT mode is weird and doesn't get reset too well. - self.flags.mode = 'BLOCK' - - - def handle_string(self, token_text): - if self.last_type == 'TK_END_EXPR' and self.flags.previous_mode in ['(COND-EXPRESSION)', '(FOR-EXPRESSION)']: - self.append(' ') - if self.last_type in ['TK_STRING', 'TK_START_BLOCK', 'TK_END_BLOCK', 'TK_SEMICOLON']: - self.append_newline() - elif self.last_type == 'TK_WORD': - self.append(' ') - - # Try to replace readable \x-encoded characters with their equivalent, - # if it is possible (e.g. '\x41\x42\x43\x01' becomes 'ABC\x01'). - def unescape(match): - block, code = match.group(0, 1) - char = chr(int(code, 16)) - if block.count('\\') == 1 and char in string.printable: - return char - return block - - token_text = re.sub(r'\\{1,2}x([a-fA-F0-9]{2})', unescape, token_text) - - self.append(token_text) - - def handle_equals(self, token_text): - if self.flags.var_line: - # just got an '=' in a var-line, different line breaking rules will apply - self.flags.var_line_tainted = True - - self.append(' ') - self.append(token_text) - self.append(' ') - - - def handle_operator(self, token_text): - space_before = True - space_after = True - - if self.flags.var_line and token_text == ',' and self.is_expression(self.flags.mode): - # do not break on comma, for ( var a = 1, b = 2 - self.flags.var_line_tainted = False - - if self.flags.var_line and token_text == ',': - if self.flags.var_line_tainted: - self.append(token_text) - self.flags.var_line_reindented = True - self.flags.var_line_tainted = False - self.append_newline() - return - else: - self.flags.var_line_tainted = False - - if self.is_special_word(self.last_text): - # return had a special handling in TK_WORD - self.append(' ') - self.append(token_text) - return - - if token_text == ':' and self.flags.in_case: - self.append(token_text) - self.append_newline() - self.flags.in_case = False - return - - if token_text == '::': - # no spaces around the exotic namespacing syntax operator - self.append(token_text) - return - - if token_text == ',': - if self.flags.var_line: - if self.flags.var_line_tainted: - # This never happens, as it's handled previously, right? - self.append(token_text) - self.append_newline() - self.flags.var_line_tainted = False - else: - self.append(token_text) - self.append(' ') - elif self.last_type == 'TK_END_BLOCK' and self.flags.mode != '(EXPRESSION)': - self.append(token_text) - if self.flags.mode == 'OBJECT' and self.last_text == '}': - self.append_newline() - else: - self.append(' ') - else: - if self.flags.mode == 'OBJECT': - self.append(token_text) - self.append_newline() - else: - # EXPR or DO_BLOCK - self.append(token_text) - self.append(' ') - # comma handled - return - elif token_text in ['--', '++', '!'] \ - or (token_text in ['+', '-'] \ - and self.last_type in ['TK_START_BLOCK', 'TK_START_EXPR', 'TK_EQUALS', 'TK_OPERATOR']) \ - or self.last_text in self.line_starters: - - space_before = False - space_after = False - - if self.last_text == ';' and self.is_expression(self.flags.mode): - # for (;; ++i) - # ^^ - space_before = True - - if self.last_type == 'TK_WORD' and self.last_text in self.line_starters: - space_before = True - - if self.flags.mode == 'BLOCK' and self.last_text in ['{', ';']: - # { foo: --i } - # foo(): --bar - self.append_newline() - - elif token_text == '.': - # decimal digits or object.property - space_before = False - - elif token_text == ':': - if self.flags.ternary_depth == 0: - self.flags.mode = 'OBJECT' - space_before = False - else: - self.flags.ternary_depth -= 1 - elif token_text == '?': - self.flags.ternary_depth += 1 - - if space_before: - self.append(' ') - - self.append(token_text) - - if space_after: - self.append(' ') - - - - def handle_block_comment(self, token_text): - - lines = token_text.replace('\x0d', '').split('\x0a') - # all lines start with an asterisk? that's a proper box comment - if not any(l for l in lines[1:] if ( l.strip() == '' or (l.lstrip())[0] != '*')): - self.append_newline() - self.append(lines[0]) - for line in lines[1:]: - self.append_newline() - self.append(' ' + line.strip()) - else: - # simple block comment: leave intact - if len(lines) > 1: - # multiline comment starts on a new line - self.append_newline() - else: - # single line /* ... */ comment stays on the same line - self.append(' ') - for line in lines: - self.append(line) - self.append('\n') - self.append_newline() - - - def handle_inline_comment(self, token_text): - self.append(' ') - self.append(token_text) - if self.is_expression(self.flags.mode): - self.append(' ') - else: - self.append_newline_forced() - - - def handle_comment(self, token_text): - if self.wanted_newline: - self.append_newline() - else: - self.append(' ') - - self.append(token_text) - self.append_newline_forced() - - - def handle_unknown(self, token_text): - if self.last_text in ['return', 'throw']: - self.append(' ') - - self.append(token_text) - - - - - -def main(): - - argv = sys.argv[1:] - - try: - opts, args = getopt.getopt(argv, "s:c:o:djbkil:htf", ['indent-size=','indent-char=','outfile=', 'disable-preserve-newlines', - 'jslint-happy', 'brace-style=', - 'keep-array-indentation', 'indent-level=', 'help', - 'usage', 'stdin', 'eval-code', 'indent-with-tabs', 'keep-function-indentation']) - except getopt.GetoptError: - return usage() - - js_options = default_options() - - file = None - outfile = 'stdout' - if len(args) == 1: - file = args[0] - - for opt, arg in opts: - if opt in ('--keep-array-indentation', '-k'): - js_options.keep_array_indentation = True - if opt in ('--keep-function-indentation','-f'): - js_options.keep_function_indentation = True - elif opt in ('--outfile', '-o'): - outfile = arg - elif opt in ('--indent-size', '-s'): - js_options.indent_size = int(arg) - elif opt in ('--indent-char', '-c'): - js_options.indent_char = arg - elif opt in ('--indent-with-tabs', '-t'): - js_options.indent_with_tabs = True - elif opt in ('--disable-preserve_newlines', '-d'): - js_options.preserve_newlines = False - elif opt in ('--jslint-happy', '-j'): - js_options.jslint_happy = True - elif opt in ('--eval-code'): - js_options.eval_code = True - elif opt in ('--brace-style', '-b'): - js_options.brace_style = arg - elif opt in ('--stdin', '-i'): - file = '-' - elif opt in ('--help', '--usage', '-h'): - return usage() - - if not file: - return usage() - else: - if outfile == 'stdout': - print(beautify_file(file, js_options)) - else: - with open(outfile, 'w') as f: - f.write(beautify_file(file, js_options) + '\n') - diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/README.specs.mkd b/mitmproxy/contrib/jsbeautifier/unpackers/README.specs.mkd deleted file mode 100644 index e937b762..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/README.specs.mkd +++ /dev/null @@ -1,25 +0,0 @@ -# UNPACKERS SPECIFICATIONS - -Nothing very difficult: an unpacker is a submodule placed in the directory -where this file was found. Each unpacker must define three symbols: - - * `PRIORITY` : integer number expressing the priority in applying this - unpacker. Lower number means higher priority. - Makes sense only if a source file has been packed with - more than one packer. - * `detect(source)` : returns `True` if source is packed, otherwise, `False`. - * `unpack(source)` : takes a `source` string and unpacks it. Must always return - valid JavaScript. That is to say, your code should look - like: - -``` -if detect(source): - return do_your_fancy_things_with(source) -else: - return source -``` - -*You can safely define any other symbol in your module, as it will be ignored.* - -`__init__` code will automatically load new unpackers, without any further step -to be accomplished. Simply drop it in this directory. diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/__init__.py b/mitmproxy/contrib/jsbeautifier/unpackers/__init__.py deleted file mode 100644 index fcb5b07a..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# General code for JSBeautifier unpackers infrastructure. See README.specs -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# - -"""General code for JSBeautifier unpackers infrastructure.""" - -import pkgutil -import re -from . import evalbased - -# NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js! -BLACKLIST = ['jsbeautifier.unpackers.evalbased'] - -class UnpackingError(Exception): - """Badly packed source or general error. Argument is a - meaningful description.""" - -def getunpackers(): - """Scans the unpackers dir, finds unpackers and add them to UNPACKERS list. - An unpacker will be loaded only if it is a valid python module (name must - adhere to naming conventions) and it is not blacklisted (i.e. inserted - into BLACKLIST.""" - path = __path__ - prefix = __name__ + '.' - unpackers = [] - interface = ['unpack', 'detect', 'PRIORITY'] - for _importer, modname, _ispkg in pkgutil.iter_modules(path, prefix): - if 'tests' not in modname and modname not in BLACKLIST: - try: - module = __import__(modname, fromlist=interface) - except ImportError: - raise UnpackingError('Bad unpacker: %s' % modname) - else: - unpackers.append(module) - - return sorted(unpackers, key = lambda mod: mod.PRIORITY) - -UNPACKERS = getunpackers() - -def run(source, evalcode=False): - """Runs the applicable unpackers and return unpacked source as a string.""" - for unpacker in [mod for mod in UNPACKERS if mod.detect(source)]: - source = unpacker.unpack(source) - if evalcode and evalbased.detect(source): - source = evalbased.unpack(source) - return source - -def filtercomments(source): - """NOT USED: strips trailing comments and put them at the top.""" - trailing_comments = [] - comment = True - - while comment: - if re.search(r'^\s*\/\*', source): - comment = source[0, source.index('*/') + 2] - elif re.search(r'^\s*\/\/', source): - comment = re.search(r'^\s*\/\/', source).group(0) - else: - comment = None - - if comment: - source = re.sub(r'^\s+', '', source[len(comment):]) - trailing_comments.append(comment) - - return '\n'.join(trailing_comments) + source diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/evalbased.py b/mitmproxy/contrib/jsbeautifier/unpackers/evalbased.py deleted file mode 100644 index b17d926e..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/evalbased.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Unpacker for eval() based packers, a part of javascript beautifier -# by Einar Lielmanis <einar@jsbeautifier.org> -# -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# usage: -# -# if detect(some_string): -# unpacked = unpack(some_string) -# - -"""Unpacker for eval() based packers: runs JS code and returns result. -Works only if a JS interpreter (e.g. Mozilla's Rhino) is installed and -properly set up on host.""" - -from subprocess import PIPE, Popen - -PRIORITY = 3 - -def detect(source): - """Detects if source is likely to be eval() packed.""" - return source.strip().lower().startswith('eval(function(') - -def unpack(source): - """Runs source and return resulting code.""" - return jseval('print %s;' % source[4:]) if detect(source) else source - -# In case of failure, we'll just return the original, without crashing on user. -def jseval(script): - """Run code in the JS interpreter and return output.""" - try: - interpreter = Popen(['js'], stdin=PIPE, stdout=PIPE) - except OSError: - return script - result, errors = interpreter.communicate(script) - if interpreter.poll() or errors: - return script - return result diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/javascriptobfuscator.py b/mitmproxy/contrib/jsbeautifier/unpackers/javascriptobfuscator.py deleted file mode 100644 index aa4344a3..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/javascriptobfuscator.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# simple unpacker/deobfuscator for scripts messed up with -# javascriptobfuscator.com -# -# written by Einar Lielmanis <einar@jsbeautifier.org> -# rewritten in Python by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# Will always return valid javascript: if `detect()` is false, `code` is -# returned, unmodified. -# -# usage: -# -# if javascriptobfuscator.detect(some_string): -# some_string = javascriptobfuscator.unpack(some_string) -# - -"""deobfuscator for scripts messed up with JavascriptObfuscator.com""" - -import re - -PRIORITY = 1 - -def smartsplit(code): - """Split `code` at " symbol, only if it is not escaped.""" - strings = [] - pos = 0 - while pos < len(code): - if code[pos] == '"': - word = '' # new word - pos += 1 - while pos < len(code): - if code[pos] == '"': - break - if code[pos] == '\\': - word += '\\' - pos += 1 - word += code[pos] - pos += 1 - strings.append('"%s"' % word) - pos += 1 - return strings - -def detect(code): - """Detects if `code` is JavascriptObfuscator.com packed.""" - # prefer `is not` idiom, so that a true boolean is returned - return (re.search(r'^var _0x[a-f0-9]+ ?\= ?\[', code) is not None) - -def unpack(code): - """Unpacks JavascriptObfuscator.com packed code.""" - if detect(code): - matches = re.search(r'var (_0x[a-f\d]+) ?\= ?\[(.*?)\];', code) - if matches: - variable = matches.group(1) - dictionary = smartsplit(matches.group(2)) - code = code[len(matches.group(0)):] - for key, value in enumerate(dictionary): - code = code.replace(r'%s[%s]' % (variable, key), value) - return code diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py b/mitmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py deleted file mode 100644 index 9893f95f..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# deobfuscator for scripts messed up with myobfuscate.com -# by Einar Lielmanis <einar@jsbeautifier.org> -# -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# usage: -# -# if detect(some_string): -# unpacked = unpack(some_string) -# - -# CAVEAT by Einar Lielmanis - -# -# You really don't want to obfuscate your scripts there: they're tracking -# your unpackings, your script gets turned into something like this, -# as of 2011-08-26: -# -# var _escape = 'your_script_escaped'; -# var _111 = document.createElement('script'); -# _111.src = 'http://api.www.myobfuscate.com/?getsrc=ok' + -# '&ref=' + encodeURIComponent(document.referrer) + -# '&url=' + encodeURIComponent(document.URL); -# var 000 = document.getElementsByTagName('head')[0]; -# 000.appendChild(_111); -# document.write(unescape(_escape)); -# - -"""Deobfuscator for scripts messed up with MyObfuscate.com""" - -import re -import base64 - -# Python 2 retrocompatibility -# pylint: disable=F0401 -# pylint: disable=E0611 -try: - from urllib import unquote -except ImportError: - from urllib.parse import unquote - -from . import UnpackingError - -PRIORITY = 1 - -CAVEAT = """// -// Unpacker warning: be careful when using myobfuscate.com for your projects: -// scripts obfuscated by the free online version call back home. -// - -""" - -SIGNATURE = (r'["\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F' - r'\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65' - r'\x66\x67\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75' - r'\x76\x77\x78\x79\x7A\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x2B' - r'\x2F\x3D","","\x63\x68\x61\x72\x41\x74","\x69\x6E\x64\x65\x78' - r'\x4F\x66","\x66\x72\x6F\x6D\x43\x68\x61\x72\x43\x6F\x64\x65","' - r'\x6C\x65\x6E\x67\x74\x68"]') - -def detect(source): - """Detects MyObfuscate.com packer.""" - return SIGNATURE in source - -def unpack(source): - """Unpacks js code packed with MyObfuscate.com""" - if not detect(source): - return source - payload = unquote(_filter(source)) - match = re.search(r"^var _escape\='<script>(.*)<\/script>'", - payload, re.DOTALL) - polished = match.group(1) if match else source - return CAVEAT + polished - -def _filter(source): - """Extracts and decode payload (original file) from `source`""" - try: - varname = re.search(r'eval\(\w+\(\w+\((\w+)\)\)\);', source).group(1) - reverse = re.search(r"var +%s *\= *'(.*)';" % varname, source).group(1) - except AttributeError: - raise UnpackingError('Malformed MyObfuscate data.') - try: - return base64.b64decode(reverse[::-1].encode('utf8')).decode('utf8') - except TypeError: - raise UnpackingError('MyObfuscate payload is not base64-encoded.') diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/packer.py b/mitmproxy/contrib/jsbeautifier/unpackers/packer.py deleted file mode 100644 index 4ada669e..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/packer.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Unpacker for Dean Edward's p.a.c.k.e.r, a part of javascript beautifier -# by Einar Lielmanis <einar@jsbeautifier.org> -# -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# usage: -# -# if detect(some_string): -# unpacked = unpack(some_string) -# - -"""Unpacker for Dean Edward's p.a.c.k.e.r""" - -import re -from . import UnpackingError - -PRIORITY = 1 - -def detect(source): - """Detects whether `source` is P.A.C.K.E.R. coded.""" - return source.replace(' ', '').startswith('eval(function(p,a,c,k,e,r') - -def unpack(source): - """Unpacks P.A.C.K.E.R. packed js code.""" - payload, symtab, radix, count = _filterargs(source) - - if count != len(symtab): - raise UnpackingError('Malformed p.a.c.k.e.r. symtab.') - - try: - unbase = Unbaser(radix) - except TypeError: - raise UnpackingError('Unknown p.a.c.k.e.r. encoding.') - - def lookup(match): - """Look up symbols in the synthetic symtab.""" - word = match.group(0) - return symtab[unbase(word)] or word - - source = re.sub(r'\b\w+\b', lookup, payload) - return _replacestrings(source) - -def _filterargs(source): - """Juice from a source file the four args needed by decoder.""" - argsregex = (r"}\('(.*)', *(\d+), *(\d+), *'(.*)'\." - r"split\('\|'\), *(\d+), *(.*)\)\)") - args = re.search(argsregex, source, re.DOTALL).groups() - - try: - return args[0], args[3].split('|'), int(args[1]), int(args[2]) - except ValueError: - raise UnpackingError('Corrupted p.a.c.k.e.r. data.') - -def _replacestrings(source): - """Strip string lookup table (list) and replace values in source.""" - match = re.search(r'var *(_\w+)\=\["(.*?)"\];', source, re.DOTALL) - - if match: - varname, strings = match.groups() - startpoint = len(match.group(0)) - lookup = strings.split('","') - variable = '%s[%%d]' % varname - for index, value in enumerate(lookup): - source = source.replace(variable % index, '"%s"' % value) - return source[startpoint:] - return source - - -class Unbaser(object): - """Functor for a given base. Will efficiently convert - strings to natural numbers.""" - ALPHABET = { - 62 : '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ', - 95 : (' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ' - '[\]^_`abcdefghijklmnopqrstuvwxyz{|}~') - } - - def __init__(self, base): - self.base = base - - # If base can be handled by int() builtin, let it do it for us - if 2 <= base <= 36: - self.unbase = lambda string: int(string, base) - else: - # Build conversion dictionary cache - try: - self.dictionary = dict((cipher, index) for - index, cipher in enumerate(self.ALPHABET[base])) - except KeyError: - raise TypeError('Unsupported base encoding.') - - self.unbase = self._dictunbaser - - def __call__(self, string): - return self.unbase(string) - - def _dictunbaser(self, string): - """Decodes a value to an integer.""" - ret = 0 - for index, cipher in enumerate(string[::-1]): - ret += (self.base ** index) * self.dictionary[cipher] - return ret diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/urlencode.py b/mitmproxy/contrib/jsbeautifier/unpackers/urlencode.py deleted file mode 100644 index 72d2bd1c..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/urlencode.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Trivial bookmarklet/escaped script detector for the javascript beautifier -# written by Einar Lielmanis <einar@jsbeautifier.org> -# rewritten in Python by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# Will always return valid javascript: if `detect()` is false, `code` is -# returned, unmodified. -# -# usage: -# -# some_string = urlencode.unpack(some_string) -# - -"""Bookmarklet/escaped script unpacker.""" - -# Python 2 retrocompatibility -# pylint: disable=F0401 -# pylint: disable=E0611 -try: - from urllib import unquote_plus -except ImportError: - from urllib.parse import unquote_plus - -PRIORITY = 0 - -def detect(code): - """Detects if a scriptlet is urlencoded.""" - # the fact that script doesn't contain any space, but has %20 instead - # should be sufficient check for now. - return ' ' not in code and ('%20' in code or code.count('%') > 3) - -def unpack(code): - """URL decode `code` source string.""" - return unquote_plus(code) if detect(code) else code diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 83f44d87..51124224 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -104,7 +104,7 @@ class DumpMaster(flow.FlowMaster): click.secho( e, file=self.options.tfile, - fg="red" if level == "error" else None, + fg=dict(error="red", warn="yellow").get(level), dim=(level == "debug"), err=(level == "error") ) @@ -118,5 +118,6 @@ class DumpMaster(flow.FlowMaster): def run(self): # pragma: no cover if self.options.rfile and not self.options.keepserving: + self.addons.done() return super(DumpMaster, self).run() diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index 3b41fe1c..6ca11b25 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -44,6 +44,12 @@ class ClientHandshakeException(TlsProtocolException): self.server = server +class InvalidServerCertificate(TlsProtocolException): + def __repr__(self): + # In contrast to most others, this is a user-facing error which needs to look good. + return str(self) + + class Socks5ProtocolException(ProtocolException): pass diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 088375fe..65a95e44 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -233,6 +233,7 @@ class FlowMaster(controller.Master): if self.server_playback: pb = self.do_server_playback(f) if not pb and self.kill_nonreplay: + self.add_log("Killed {}".format(f.request.url), "info") f.kill(self) def replay_request(self, f, block=False): diff --git a/mitmproxy/flow/state.py b/mitmproxy/flow/state.py index 4287d77b..efcb2d89 100644 --- a/mitmproxy/flow/state.py +++ b/mitmproxy/flow/state.py @@ -191,7 +191,7 @@ class State(object): self.intercept = None @property - def limit_txt(self): + def filter_txt(self): return getattr(self.view.filt, "pattern", None) def flow_count(self): @@ -225,8 +225,8 @@ class State(object): def load_flows(self, flows): self.flows._extend(flows) - def set_limit(self, txt): - if txt == self.limit_txt: + def set_view_filter(self, txt): + if txt == self.filter_txt: return if txt: f = filt.parse(txt) diff --git a/mitmproxy/main.py b/mitmproxy/main.py index 6d44108e..6ae99bdd 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/main.py @@ -68,7 +68,7 @@ def mitmproxy(args=None): # pragma: no cover console_options.eventlog = args.eventlog console_options.follow = args.follow console_options.intercept = args.intercept - console_options.limit = args.limit + console_options.filter = args.filter console_options.no_mouse = args.no_mouse server = process_options(parser, console_options, args) @@ -92,6 +92,7 @@ def mitmdump(args=None): # pragma: no cover if args.quiet: args.flow_detail = 0 + master = None try: dump_options = dump.Options(**cmdline.get_common_options(args)) dump_options.flow_detail = args.flow_detail @@ -110,7 +111,7 @@ def mitmdump(args=None): # pragma: no cover sys.exit(1) except (KeyboardInterrupt, _thread.error): pass - if master.has_errored: + if master is None or master.has_errored: print("mitmdump: errors occurred during run", file=sys.stderr) sys.exit(1) diff --git a/mitmproxy/models/http.py b/mitmproxy/models/http.py index 7781e61f..d56eb29a 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/models/http.py @@ -225,7 +225,7 @@ class HTTPFlow(Flow): def make_error_response(status_code, message, headers=None): - response = status_codes.RESPONSES.get(status_code, "Unknown").encode() + response = status_codes.RESPONSES.get(status_code, "Unknown") body = """ <html> <head> diff --git a/mitmproxy/onboarding/app.py b/mitmproxy/onboarding/app.py index e26efae8..491ddbfe 100644 --- a/mitmproxy/onboarding/app.py +++ b/mitmproxy/onboarding/app.py @@ -48,6 +48,7 @@ class PEM(tornado.web.RequestHandler): def get(self): p = os.path.join(self.request.master.options.cadir, self.filename) + p = os.path.expanduser(p) self.set_header("Content-Type", "application/x-x509-ca-cert") self.set_header( "Content-Disposition", diff --git a/mitmproxy/options.py b/mitmproxy/options.py index bdc0db4e..75798381 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -73,7 +73,7 @@ class Options(optmanager.OptManager): upstream_auth = "", # type: str ssl_version_client="secure", # type: str ssl_version_server="secure", # type: str - ssl_verify_upstream_cert=False, # type: bool + ssl_insecure=False, # type: bool ssl_verify_upstream_trusted_cadir=None, # type: str ssl_verify_upstream_trusted_ca=None, # type: str tcp_hosts = (), # type: Sequence[str] @@ -130,7 +130,7 @@ class Options(optmanager.OptManager): self.upstream_auth = upstream_auth self.ssl_version_client = ssl_version_client self.ssl_version_server = ssl_version_server - self.ssl_verify_upstream_cert = ssl_verify_upstream_cert + self.ssl_insecure = ssl_insecure self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca self.tcp_hosts = tcp_hosts diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 140c7ca8..92d32b2d 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -86,7 +86,10 @@ class OptManager(object): """ if attr not in self._opts: raise KeyError("No such option: %s" % attr) - return lambda x: self.__setattr__(attr, x) + + def setter(x): + setattr(self, attr, x) + return setter def toggler(self, attr): """ @@ -95,7 +98,10 @@ class OptManager(object): """ if attr not in self._opts: raise KeyError("No such option: %s" % attr) - return lambda: self.__setattr__(attr, not getattr(self, attr)) + + def toggle(): + setattr(self, attr, not getattr(self, attr)) + return toggle def __repr__(self): options = pprint.pformat(self._opts, indent=4).strip(" {}") diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py index e1ff7c47..2e350131 100644 --- a/mitmproxy/platform/__init__.py +++ b/mitmproxy/platform/__init__.py @@ -1,8 +1,9 @@ import sys +import re resolver = None -if sys.platform == "linux2": +if re.match(r"linux(?:2)?", sys.platform): from . import linux resolver = linux.Resolver elif sys.platform == "darwin": diff --git a/mitmproxy/protocol/tls.py b/mitmproxy/protocol/tls.py index 51f4d80d..d08e2e32 100644 --- a/mitmproxy/protocol/tls.py +++ b/mitmproxy/protocol/tls.py @@ -543,25 +543,12 @@ class TlsLayer(base.Layer): ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") + self.log(str(tls_cert_err), "warn") + self.log("Ignoring server verification error, continuing with connection", "warn") except netlib.exceptions.InvalidCertificateException as e: - tls_cert_err = self.server_conn.ssl_verification_error - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") six.reraise( - exceptions.TlsProtocolException, - exceptions.TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( - address=repr(self.server_conn.address), - sni=self.server_sni, - e=repr(e), - )), + exceptions.InvalidServerCertificate, + exceptions.InvalidServerCertificate(str(e)), sys.exc_info()[2] ) except netlib.exceptions.TlsException as e: diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index a74ba7e2..cf75830a 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -83,24 +83,18 @@ class ProxyConfig: options.changed.connect(self.configure) def configure(self, options, updated): - conflict = all( - [ - options.add_upstream_certs_to_client_chain, - options.ssl_verify_upstream_cert - ] - ) - if conflict: + # type: (mitmproxy.options.Options, Any) -> None + if options.add_upstream_certs_to_client_chain and not options.ssl_insecure: raise exceptions.OptionsError( - "The verify-upstream-cert and add-upstream-certs-to-client-chain " - "options are mutually exclusive. If upstream certificates are verified " - "then extra upstream certificates are not available for inclusion " - "to the client chain." + "The verify-upstream-cert requires certificate verification to be disabled. " + "If upstream certificates are verified then extra upstream certificates are " + "not available for inclusion to the client chain." ) - if options.ssl_verify_upstream_cert: - self.openssl_verification_mode_server = SSL.VERIFY_PEER - else: + if options.ssl_insecure: self.openssl_verification_mode_server = SSL.VERIFY_NONE + else: + self.openssl_verification_mode_server = SSL.VERIFY_PEER self.check_ignore = HostMatcher(options.ignore_hosts) self.check_tcp = HostMatcher(options.tcp_hosts) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 26f2e294..4fd5755a 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -125,11 +125,14 @@ class ConnectionHandler(object): self.log( "Client Handshake failed. " "The client may not trust the proxy's certificate for {}.".format(e.server), - "error" + "warn" ) self.log(repr(e), "debug") + elif isinstance(e, exceptions.InvalidServerCertificate): + self.log(str(e), "warn") + self.log("Invalid certificate, closing connection. Pass --insecure to disable validation.", "warn") else: - self.log(repr(e), "info") + self.log(repr(e), "warn") self.log(traceback.format_exc(), "debug") # If an error propagates to the topmost level, diff --git a/mitmproxy/web/static/images/favicon.ico b/mitmproxy/web/static/images/favicon.ico Binary files differnew file mode 100644 index 00000000..bfd2fde7 --- /dev/null +++ b/mitmproxy/web/static/images/favicon.ico diff --git a/mitmproxy/web/templates/index.html b/mitmproxy/web/templates/index.html index 165d7d3d..db9d2ecb 100644 --- a/mitmproxy/web/templates/index.html +++ b/mitmproxy/web/templates/index.html @@ -6,10 +6,11 @@ <meta name="viewport" content="width=device-width, initial-scale=1"> <link rel="stylesheet" href="/static/vendor.css"/> <link rel="stylesheet" href="/static/app.css"/> + <link rel="icon" href="/static/images/favicon.ico" type="image/x-icon"/> <script src="/static/vendor.js"></script> <script src="/static/app.js"></script> </head> <body> <div id="mitmproxy"></div> </body> -</html>
\ No newline at end of file +</html> diff --git a/netlib/encoding.py b/netlib/encoding.py index da282194..9c8acff7 100644 --- a/netlib/encoding.py +++ b/netlib/encoding.py @@ -8,6 +8,7 @@ import collections from io import BytesIO import gzip import zlib +import brotli from typing import Union # noqa @@ -45,7 +46,7 @@ def decode(encoded, encoding, errors='strict'): decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate"): + if encoding in ("gzip", "deflate", "br"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except Exception as e: @@ -81,7 +82,7 @@ def encode(decoded, encoding, errors='strict'): encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate"): + if encoding in ("gzip", "deflate", "br"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except Exception as e: @@ -113,6 +114,14 @@ def encode_gzip(content): return s.getvalue() +def decode_brotli(content): + return brotli.decompress(content) + + +def encode_brotli(content): + return brotli.compress(content) + + def decode_deflate(content): """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -139,11 +148,13 @@ custom_decode = { "identity": identity, "gzip": decode_gzip, "deflate": decode_deflate, + "br": decode_brotli, } custom_encode = { "identity": identity, "gzip": encode_gzip, "deflate": encode_deflate, + "br": encode_brotli, } __all__ = ["encode", "decode"] diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index fdf4ef8f..436b5965 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -9,7 +9,8 @@ from netlib.http import http1, http2, status_codes, multipart __all__ = [ "Request", "Response", + "Message", "Headers", "parse_content_type", "decoded", "http1", "http2", "status_codes", "multipart", -]
\ No newline at end of file +] diff --git a/netlib/http/message.py b/netlib/http/message.py index be35b8d1..ce92bab1 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -248,7 +248,7 @@ class Message(basetypes.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate" or "identity". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/netlib/http/request.py b/netlib/http/request.py index 061217a3..d59fead4 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -337,7 +337,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate"} + for e in {"gzip", "identity", "deflate", "br"} if e in accept_encoding ) ) diff --git a/netlib/strutils.py b/netlib/strutils.py index 8f27ebb7..4a46b6b1 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -69,7 +69,7 @@ def escape_control_characters(text, keep_spacing=True): return text.translate(trans) -def bytes_to_escaped_str(data, keep_spacing=False): +def bytes_to_escaped_str(data, keep_spacing=False, escape_single_quotes=False): """ Take bytes and return a safe string that can be displayed to the user. @@ -86,6 +86,8 @@ def bytes_to_escaped_str(data, keep_spacing=False): # We always insert a double-quote here so that we get a single-quoted string back # https://stackoverflow.com/questions/29019340/why-does-python-use-different-quotes-for-representing-strings-depending-on-their ret = repr(b'"' + data).lstrip("b")[2:-1] + if not escape_single_quotes: + ret = re.sub(r"(?<!\\)(\\\\)*\\'", lambda m: (m.group(1) or "") + "'", ret) if keep_spacing: ret = re.sub( r"(?<!\\)(\\\\)*\\([nrt])", diff --git a/netlib/tcp.py b/netlib/tcp.py index cf099edd..e5c84165 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -8,6 +8,10 @@ import time import traceback import binascii + +from typing import Optional # noqa + +from netlib import strutils from six.moves import range import certifi @@ -35,7 +39,7 @@ EINTR = 4 if os.environ.get("NO_ALPN"): HAS_ALPN = False else: - HAS_ALPN = OpenSSL._util.lib.Cryptography_HAS_ALPN + HAS_ALPN = SSL._lib.Cryptography_HAS_ALPN # To enable all SSL methods use: SSLv23 # then add options to disable certain methods @@ -287,16 +291,7 @@ class Reader(_FileLike): raise exceptions.TcpException(repr(e)) elif isinstance(self.o, SSL.Connection): try: - if tuple(int(x) for x in OpenSSL.__version__.split(".")[:2]) > (0, 15): - return self.o.recv(length, socket.MSG_PEEK) - else: - # TODO: remove once a new version is released - # Polyfill for pyOpenSSL <= 0.15.1 - # Taken from https://github.com/pyca/pyopenssl/commit/1d95dea7fea03c7c0df345a5ea30c12d8a0378d2 - buf = SSL._ffi.new("char[]", length) - result = SSL._lib.SSL_peek(self.o._ssl, buf, length) - self.o._raise_ssl_error(self.o._ssl, result) - return SSL._ffi.buffer(buf, result)[:] + return self.o.recv(length, socket.MSG_PEEK) except SSL.Error as e: six.reraise(exceptions.TlsException, exceptions.TlsException(str(e)), sys.exc_info()[2]) else: @@ -511,6 +506,7 @@ class _Connection(object): alpn_protos=None, alpn_select=None, alpn_select_callback=None, + sni=None, ): """ Creates an SSL Context. @@ -532,8 +528,14 @@ class _Connection(object): if verify_options is not None: def verify_cert(conn, x509, errno, err_depth, is_cert_verified): if not is_cert_verified: - self.ssl_verification_error = dict(errno=errno, - depth=err_depth) + self.ssl_verification_error = exceptions.InvalidCertificateException( + "Certificate Verification Error for {}: {} (errno: {}, depth: {})".format( + sni, + strutils.native(SSL._ffi.string(SSL._lib.X509_verify_cert_error_string(errno)), "utf8"), + errno, + err_depth + ) + ) return is_cert_verified context.set_verify(verify_options, verify_cert) @@ -609,7 +611,7 @@ class TCPClient(_Connection): self.source_address = source_address self.cert = None self.server_certs = [] - self.ssl_verification_error = None + self.ssl_verification_error = None # type: Optional[exceptions.InvalidCertificateException] self.sni = None @property @@ -671,6 +673,7 @@ class TCPClient(_Connection): context = self.create_ssl_context( alpn_protos=alpn_protos, + sni=sni, **sslctx_kwargs ) self.connection = SSL.Connection(context, self.connection) @@ -682,14 +685,14 @@ class TCPClient(_Connection): self.connection.do_handshake() except SSL.Error as v: if self.ssl_verification_error: - raise exceptions.InvalidCertificateException("SSL handshake error: %s" % repr(v)) + raise self.ssl_verification_error else: raise exceptions.TlsException("SSL handshake error: %s" % repr(v)) else: # Fix for pre v1.0 OpenSSL, which doesn't throw an exception on # certificate validation failure - if verification_mode == SSL.VERIFY_PEER and self.ssl_verification_error is not None: - raise exceptions.InvalidCertificateException("SSL handshake error: certificate verify failed") + if verification_mode == SSL.VERIFY_PEER and self.ssl_verification_error: + raise self.ssl_verification_error self.cert = certutils.SSLCert(self.connection.get_peer_certificate()) @@ -710,9 +713,14 @@ class TCPClient(_Connection): hostname = "no-hostname" ssl_match_hostname.match_hostname(crt, hostname) except (ValueError, ssl_match_hostname.CertificateError) as e: - self.ssl_verification_error = dict(depth=0, errno="Invalid Hostname") + self.ssl_verification_error = exceptions.InvalidCertificateException( + "Certificate Verification Error for {}: {}".format( + sni or repr(self.address), + str(e) + ) + ) if verification_mode == SSL.VERIFY_PEER: - raise exceptions.InvalidCertificateException("Presented certificate for {} is not valid: {}".format(sni, str(e))) + raise self.ssl_verification_error self.ssl_established = True self.rfile.set_descriptor(self.connection) diff --git a/pathod/language/base.py b/pathod/language/base.py index 25f3fd1a..39155858 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -136,7 +136,7 @@ class TokValueLiteral(_TokValueLiteral): def spec(self): inner = strutils.bytes_to_escaped_str(self.val) - inner = inner.replace(r"\'", r"\x27") + inner = inner.replace(r"'", r"\x27") return "'" + inner + "'" @@ -148,7 +148,7 @@ class TokValueNakedLiteral(_TokValueLiteral): return e.setParseAction(lambda x: cls(*x)) def spec(self): - return strutils.bytes_to_escaped_str(self.val) + return strutils.bytes_to_escaped_str(self.val, escape_single_quotes=True) class TokValueGenerate(Token): @@ -166,7 +166,7 @@ class TokValueGenerate(Token): def freeze(self, settings): g = self.get_generator(settings) - return TokValueLiteral(strutils.bytes_to_escaped_str(g[:])) + return TokValueLiteral(strutils.bytes_to_escaped_str(g[:], escape_single_quotes=True)) @classmethod def expr(cls): @@ -578,4 +578,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode()))) + return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True))) diff --git a/pathod/pathoc.py b/pathod/pathoc.py index c6783878..5831ba3e 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -444,7 +444,7 @@ class Pathoc(tcp.TCPClient): finally: if resp: lg("<< %s %s: %s bytes" % ( - resp.status_code, strutils.bytes_to_escaped_str(resp.reason.encode()), len(resp.content) + resp.status_code, strutils.escape_control_characters(resp.reason) if resp.reason else "", len(resp.content) )) if resp.status_code in self.ignorecodes: lg.suppress() diff --git a/release/rtool.py b/release/rtool.py index 4e43eaef..45e1416e 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,29 +1,30 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import absolute_import, print_function, division -from os.path import join + import contextlib +import fnmatch import os +import platform +import runpy +import shlex import shutil import subprocess -import re -import shlex -import runpy -import zipfile +import sys import tarfile -import platform +import zipfile +from os.path import join, abspath, normpath, dirname, exists, basename + import click import pysftp -import fnmatch # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -import sys - if platform.system() == "Windows": VENV_BIN = "Scripts" else: VENV_BIN = "bin" +# ZipFile and tarfile have slightly different APIs. Fix that. if platform.system() == "Windows": def Archive(name): a = zipfile.ZipFile(name, "w") @@ -33,13 +34,13 @@ else: def Archive(name): return tarfile.open(name, "w:gz") -RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) -DIST_DIR = join(RELEASE_DIR, "dist") -ROOT_DIR = os.path.normpath(join(RELEASE_DIR, "..")) -RELEASE_SPEC_DIR = join(RELEASE_DIR, "specs") -VERSION_FILE = join(ROOT_DIR, "netlib/version.py") +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") PYINSTALLER_DIST = join(BUILD_DIR, "binaries") @@ -47,27 +48,35 @@ VENV_DIR = join(BUILD_DIR, "venv") VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") -project = { - "name": "mitmproxy", - "tools": ["pathod", "pathoc", "mitmproxy", "mitmdump", "mitmweb"], - "bdists": { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] - }, - "dir": ROOT_DIR, - "python_version": "py2.py3", +# Project Configuration +VERSION_FILE = join(ROOT_DIR, "netlib", "version.py") +PROJECT_NAME = "mitmproxy" +PYTHON_VERSION = "py2.py3" +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] } if platform.system() == "Windows": - project["tools"].remove("mitmproxy") - project["bdists"]["mitmproxy"].remove("mitmproxy") + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in BDISTS.values() + for tool in tools +] -def get_version(): +def get_version() -> str: return runpy.run_path(VERSION_FILE)["VERSION"] -def get_snapshot_version(): - last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit(b"-", 2) +def git(args: str) -> str: + with chdir(ROOT_DIR): + return subprocess.check_output(["git"] + shlex.split(args)).decode() + + +def get_snapshot_version() -> str: + last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) tag_dist = int(tag_dist) if tag_dist == 0: return get_version() @@ -76,11 +85,11 @@ def get_snapshot_version(): return "{version}dev{tag_dist:04}-0x{commit}".format( version=get_version(), # this should already be the next version tag_dist=tag_dist, - commit=commit.decode() + commit=commit ) -def archive_name(project): +def archive_name(bdist: str) -> str: platform_tag = { "Darwin": "osx", "Windows": "win32", @@ -91,18 +100,18 @@ def archive_name(project): else: ext = "tar.gz" return "{project}-{version}-{platform}.{ext}".format( - project=project, + project=bdist, version=get_version(), platform=platform_tag, ext=ext ) -def wheel_name(): +def wheel_name() -> str: return "{project}-{version}-{py_version}-none-any.whl".format( - project=project["name"], + project=PROJECT_NAME, version=get_version(), - py_version=project["python_version"] + py_version=PYTHON_VERSION ) @@ -119,18 +128,13 @@ def empty_pythonpath(): @contextlib.contextmanager -def chdir(path): +def chdir(path: str): old_dir = os.getcwd() os.chdir(path) yield os.chdir(old_dir) -def git(args): - with chdir(ROOT_DIR): - return subprocess.check_output(["git"] + shlex.split(args)) - - @click.group(chain=True) def cli(): """ @@ -147,95 +151,79 @@ def contributors(): with chdir(ROOT_DIR): print("Updating CONTRIBUTORS...") contributors_data = git("shortlog -n -s") - with open("CONTRIBUTORS", "w") as f: - f.write(contributors_data) + with open("CONTRIBUTORS", "wb") as f: + f.write(contributors_data.encode()) -@cli.command("set-version") -@click.argument('version') -def set_version(version): +@cli.command("wheel") +def make_wheel(): """ - Update version information - """ - print("Update versions...") - version = ", ".join(version.split(".")) - print("Update %s..." % VERSION_FILE) - with open(VERSION_FILE, "rb") as f: - content = f.read() - new_content = re.sub( - r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, - content - ) - with open(VERSION_FILE, "wb") as f: - f.write(new_content) - - -@cli.command("wheels") -def wheels(): - """ - Build wheels + Build wheel """ with empty_pythonpath(): - print("Building release...") - if os.path.exists(DIST_DIR): + if exists(DIST_DIR): shutil.rmtree(DIST_DIR) - print("Creating wheel for %s ..." % project["name"]) + print("Creating wheel...") subprocess.check_call( [ "python", "./setup.py", "-q", "bdist_wheel", "--dist-dir", DIST_DIR, "--universal" ], - cwd=project["dir"] + cwd=ROOT_DIR ) print("Creating virtualenv for test install...") - if os.path.exists(VENV_DIR): + if exists(VENV_DIR): shutil.rmtree(VENV_DIR) subprocess.check_call(["virtualenv", "-q", VENV_DIR]) with chdir(DIST_DIR): - print("Installing %s..." % project["name"]) + print("Install wheel into virtualenv...") # lxml... if platform.system() == "Windows" and sys.version_info[0] == 3: - subprocess.check_call([VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]) + subprocess.check_call( + [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"] + ) subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()]) - print("Running binaries...") - for tool in project["tools"]: + print("Running tools...") + for tool in TOOLS: tool = join(VENV_DIR, VENV_BIN, tool) print("> %s --version" % tool) - print(subprocess.check_output([tool, "--version"])) + print(subprocess.check_output([tool, "--version"]).decode()) print("Virtualenv available for further testing:") - print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate"))) + print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate"))) @cli.command("bdist") -@click.option("--use-existing-wheels/--no-use-existing-wheels", default=False) +@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False) @click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.1.1") +@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1") @click.pass_context -def bdist(ctx, use_existing_wheels, pyinstaller_version): +def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version): """ Build a binary distribution """ - if os.path.exists(PYINSTALLER_TEMP): + if exists(PYINSTALLER_TEMP): shutil.rmtree(PYINSTALLER_TEMP) - if os.path.exists(PYINSTALLER_DIST): + if exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) - if not use_existing_wheels: - ctx.invoke(wheels) + if not use_existing_wheel: + ctx.invoke(make_wheel) - print("Installing PyInstaller...") - subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version]) + print("Installing PyInstaller and setuptools...") + subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version]) + print(subprocess.check_output([VENV_PIP, "freeze"]).decode()) - for bdist_project, tools in project["bdists"].items(): - with Archive(join(DIST_DIR, archive_name(bdist_project))) as archive: + for bdist, tools in BDISTS.items(): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: for tool in tools: # This is PyInstaller, so it messes up paths. # We need to make sure that we are in the spec folder. - with chdir(RELEASE_SPEC_DIR): + with chdir(PYINSTALLER_SPEC): print("Building %s binary..." % tool) subprocess.check_call( [ @@ -255,10 +243,10 @@ def bdist(ctx, use_existing_wheels, pyinstaller_version): if platform.system() == "Windows": executable += ".exe" print("> %s --version" % executable) - subprocess.check_call([executable, "--version"]) + print(subprocess.check_output([executable, "--version"]).decode()) - archive.add(executable, os.path.basename(executable)) - print("Packed {}.".format(archive_name(bdist_project))) + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) @cli.command("upload-release") @@ -298,89 +286,49 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, username=user, private_key=private_key, private_key_pass=private_key_password) as sftp: - - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - for bdist in project["bdists"].keys(): + dir_name = "snapshots/v{}".format(get_version()) + sftp.makedirs(dir_name) + with sftp.cd(dir_name): + files = [] + if wheel: + files.append(wheel_name()) + if bdist: + for bdist in BDISTS.keys(): files.append(archive_name(bdist)) - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = f.replace(get_version(), get_snapshot_version()) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - # We hide the file during upload. - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) - - -@cli.command("wizard") -@click.option('--next-version', prompt=True) -@click.option('--username', prompt="PyPI Username") -@click.password_option(confirmation_prompt=False, prompt="PyPI Password") -@click.option('--repository', default="pypi") -@click.pass_context -def wizard(ctx, next_version, username, password, repository): - """ - Interactive Release Wizard - """ - is_dirty = git("status --porcelain") - if is_dirty: - raise RuntimeError("Repository is not clean.") - - # update contributors file - ctx.invoke(contributors) - - # Build test release - ctx.invoke(bdist) - - try: - click.confirm("Please test the release now. Is it ok?", abort=True) - except click.Abort: - # undo changes - git("checkout CONTRIBUTORS") - raise - - # Everything ok - let's ship it! - git("tag v{}".format(get_version())) - git("push --tags") - ctx.invoke( - upload_release, - username=username, password=password, repository=repository - ) - - click.confirm("Now please wait until CI has built binaries. Finished?") - - # version bump commit - ctx.invoke(set_version, version=next_version) - git("commit -a -m \"bump version\"") - git("push") + for f in files: + local_path = join(DIST_DIR, f) + remote_filename = f.replace(get_version(), get_snapshot_version()) + symlink_path = "../{}".format(f.replace(get_version(), "latest")) + + # Upload new version + print("Uploading {} as {}...".format(f, remote_filename)) + with click.progressbar(length=os.stat(local_path).st_size) as bar: + # We hide the file during upload + sftp.put( + local_path, + "." + remote_filename, + callback=lambda done, total: bar.update(done - bar.pos) + ) - click.echo("All done!") + # Delete old versions + old_version = f.replace(get_version(), "*") + for f_old in sftp.listdir(): + if fnmatch.fnmatch(f_old, old_version): + print("Removing {}...".format(f_old)) + sftp.remove(f_old) + + # Show new version + sftp.rename("." + remote_filename, remote_filename) + + # update symlink for the latest release + if sftp.lexists(symlink_path): + print("Removing {}...".format(symlink_path)) + sftp.remove(symlink_path) + if f != wheel_name(): + # "latest" isn't a proper wheel version, so this could not be installed. + # https://github.com/mitmproxy/mitmproxy/issues/1065 + sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) if __name__ == "__main__": @@ -67,10 +67,12 @@ setup( "configargparse>=0.10, <0.11", "construct>=2.5.2, <2.6", "cryptography>=1.3, <1.5", + "cssutils>=1.0.1, <1.1", "Flask>=0.10.1, <0.12", "h2>=2.4.0, <3", "html2text>=2016.1.8, <=2016.5.29", "hyperframe>=4.0.1, <5", + "jsbeautifier>=1.6.3, <1.7", "lxml>=3.5.0, <=3.6.0", # no wheels for 3.6.1 yet. "Pillow>=3.2, <3.4", "passlib>=1.6.5, <1.7", @@ -83,6 +85,7 @@ setup( "tornado>=4.3, <4.5", "urwid>=1.3.1, <1.4", "watchdog>=0.8.3, <0.9", + "brotlipy>=0.3.0, <0.4", ], extras_require={ ':sys_platform == "win32"': [ @@ -110,7 +113,6 @@ setup( "sphinx_rtd_theme>=0.1.9, <0.2", ], 'contentviews': [ - "cssutils>=1.0.1, <1.1", # TODO: Find Python 3 replacements # "protobuf>=2.6.1, <2.7", # "pyamf>=0.8.0, <0.9", diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py index 2d551bab..1c7173e0 100644 --- a/test/mitmproxy/builtins/test_dumper.py +++ b/test/mitmproxy/builtins/test_dumper.py @@ -81,4 +81,4 @@ class TestContentView(mastertest.MasterTest): d = dumper.Dumper() m.addons.add(o, d) self.invoke(m, "response", tutils.tflow()) - assert "Content viewer failed" in m.event_log[0][1]
\ No newline at end of file + assert "Content viewer failed" in m.event_log[0][1] diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index b84e4c1c..fcb87e1b 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -76,7 +76,7 @@ class TestConsoleState: self._add_response(c) self._add_request(c) self._add_response(c) - assert not c.set_limit("~s") + assert not c.set_view_filter("~s") assert len(c.view) == 3 assert c.focus == 0 diff --git a/test/mitmproxy/data/servercert/9da13359.0 b/test/mitmproxy/data/servercert/9da13359.0 new file mode 100644 index 00000000..b22e4d20 --- /dev/null +++ b/test/mitmproxy/data/servercert/9da13359.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAPAfPQGCV/Z4MA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTUxMTAxMTY0ODAxWhcNMTgwODIxMTY0ODAxWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEArp8LD34JhKCwcQbwIYQMg4+eCgLVN8fwB7+/qOfJbArPs0djFBN+F7c6 +HGvMr24BKUk5u8pn4dPtNurm/vPC8ovNGmcXz62BQJpcMX2veVdRsF7yNwhNacNJ +Arq+70zNMwYBznx0XUxMF6j6nVFf3AW6SU04ylT4Mp3SY/BUUDAdfl1eRo0mPLNS +8rpsN+8YBw1Q7SCuBRVqpOgVIsL88svgQUSOlzvMZPBpG/cmB3BNKNrltwb5iFEI +1jAV7uSj5IcIuNO/246kfsDVPTFMJIzav/CUoidd5UNw+SoFDlzh8sA7L1Bm7D1/ +3KHYSKswGsSR3kynAl10w/SJKDtn8wIDAQABo1AwTjAdBgNVHQ4EFgQUgOcrtxBX +LxbpnOT65d+vpfyWUkgwHwYDVR0jBBgwFoAUgOcrtxBXLxbpnOT65d+vpfyWUkgw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAEE9bFmUCA+6cvESKPoi2 +TGSpV652d0xd2U66LpEXeiWRJFLz8YGgoJCx3QFGBscJDXxrLxrBBBV/tCpEqypo +pYIqsawH7M66jpOr83Us3M8JC2eFBZJocMpXxdytWqHik5VKZNx6VQFT8bS7+yVC +VoUKePhlgcg+pmo41qjqieBNKRMh/1tXS77DI1lgO5wZLVrLXcdqWuDpmaQOKJeq +G/nxytCW/YJA7bFn/8Gjy8DYypJSeeaKu7o3P3+ONJHdIMHb+MdcheDBS9AOFSeo +xI0D5EbO9F873O77l7nbD7B0X34HFN0nGczC4poexIpbDFG3hAPekwZ5KC6VwJLc +1Q== +-----END CERTIFICATE----- diff --git a/test/mitmproxy/data/servercert/self-signed.pem b/test/mitmproxy/data/servercert/self-signed.pem new file mode 100644 index 00000000..cd066a24 --- /dev/null +++ b/test/mitmproxy/data/servercert/self-signed.pem @@ -0,0 +1,46 @@ +-----BEGIN CERTIFICATE----- +MIIDEzCCAfugAwIBAgIJAJ945xt1FRsfMA0GCSqGSIb3DQEBCwUAMCAxHjAcBgNV +BAMMFWV4YW1wbGUubWl0bXByb3h5Lm9yZzAeFw0xNTExMDExNjQ4MDJaFw0xODA4 +MjExNjQ4MDJaMCAxHjAcBgNVBAMMFWV4YW1wbGUubWl0bXByb3h5Lm9yZzCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALFxyzPfjgIghOMMnJlW80yB84xC +nJtko3tuyOdozgTCyha2W+NdIKPNZJtWrzN4P0B5PlozCDwfcSYffLs0WZs8LRWv +BfZX8+oX+14qQjKFsiqgO65cTLP3qlPySYPJQQ37vOP1Y5Yf8nQq2mwQdC18hLtT +QOANG6OFoSplpBLsYF+QeoMgqCTa6hrl/5GLmQoDRTjXkv3Sj379AUDMybuBqccm +q5EIqCrE4+xJ8JywJclAVn2YP14baiFrrYCsYYg4sS1Od6xFj+xtpLe7My3AYjB9 +/aeHd8vDiob0cqOW1TFwhqgJKuErfFyg8lZ2hJmStJKyfofWuY/gl/vnvX0CAwEA +AaNQME4wHQYDVR0OBBYEFB8d32zK8eqZIoKw4jXzYzhw4amPMB8GA1UdIwQYMBaA +FB8d32zK8eqZIoKw4jXzYzhw4amPMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAJmo2oKv1OEjZ0Q4yELO6BAnHAkmBKpW+zmLyQa8idxtLVkI9uXk3iqY +GWugkmcUZCTVFRWv/QXQQSex+00IY3x2rdHbtuZwcyKiz2u8WEmfW1rOIwBaFJ1i +v7+SA2aZs6vepN2sE56X54c/YbwQooaKZtOb+djWXYMJrc/Ezj0J7oQIJTptYV8v +/3216yCHRp/KCL7yTLtiw25xKuXNu/gkcd8wZOY9rS2qMUD897MJF0MvgJoauRBd +d4XEYCNKkrIRmfqrkiRQfAZpvpoutH6NCk7KuQYcI0BlOHlsnHHcs/w72EEqHwFq +x6476tW/t8GJDZVD74+pNBcLifXxArE= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAsXHLM9+OAiCE4wycmVbzTIHzjEKcm2Sje27I52jOBMLKFrZb +410go81km1avM3g/QHk+WjMIPB9xJh98uzRZmzwtFa8F9lfz6hf7XipCMoWyKqA7 +rlxMs/eqU/JJg8lBDfu84/Vjlh/ydCrabBB0LXyEu1NA4A0bo4WhKmWkEuxgX5B6 +gyCoJNrqGuX/kYuZCgNFONeS/dKPfv0BQMzJu4GpxyarkQioKsTj7EnwnLAlyUBW +fZg/XhtqIWutgKxhiDixLU53rEWP7G2kt7szLcBiMH39p4d3y8OKhvRyo5bVMXCG +qAkq4St8XKDyVnaEmZK0krJ+h9a5j+CX++e9fQIDAQABAoIBAQCT+FvGbych2PJX +0D2KlXqgE0IAdc/YuYymstSwPLKIP9N8KyfnKtK8Jdw+uYOyfRTp8/EuEJ5OXL3j +V6CRD++lRwIlseVb7y5EySjh9oVrUhgn+aSrGucPsHkGNeZeEmbAfWugARLBrvRl +MRMhyHrJL6wT9jIEZInmy9mA3G99IuFW3rS8UR1Yu7zyvhtjvop1xg/wfEUu24Ty +PvMfnwaDcZHCz2tmu2KJvaxSBAG3FKmAqeMvk1Gt5m2keKgw03M+EX0LrM8ybWqn +VwB8tnSyMBLVFLIXMpIiSfpji10+p9fdKFMRF++D6qVwyoxPiIq+yEJapxXiqLea +mkhtJW91AoGBAOvIb7bZvH4wYvi6txs2pygF3ZMjqg/fycnplrmYMrjeeDeeN4v1 +h/5tkN9TeTkHRaN3L7v49NEUDhDyuopLTNfWpYdv63U/BVzvgMm/guacTYkx9whB +OvQ2YekR/WKg7kuyrTZidTDz+mjU+1b8JaWGjiDc6vFwxZA7uWicaGGHAoGBAMCo +y/2AwFGwCR+5bET1nTTyxok6iKo4k6R/7DJe4Bq8VLifoyX3zDlGG/33KN3xVqBU +xnT9gkii1lfX2U+4iM+GOSPl0nG0hOEqEH+vFHszpHybDeNez3FEyIbgOzg6u7sV +NOy+P94L5EMQVEmWp5g6Vm3k9kr92Bd9UacKQPnbAoGAMN8KyMu41i8RVJze9zUM +0K7mjmkGBuRL3x4br7xsRwVVxbF1sfzig0oSjTewGLH5LTi3HC8uD2gowjqNj7yr +4NEM3lXEaDj305uRBkA70bD0IUvJ+FwM7DGZecXQz3Cr8+TFIlCmGc94R+Jddlot +M3IAY69mw0SsroiylYxV1mECgYAcSGtx8rXJCDO+sYTgdsI2ZLGasbogax/ZlWIC +XwU9R4qUc/MKft8/RTiUxvT76BMUhH2B7Tl0GlunF6vyVR/Yf1biGzoSsTKUr40u +gXBbSdCK7mRSjbecZEGf80keTxkCNPHJE4DiwxImej41c2V1JpNLnMI/bhaMFDyp +bgrt4wKBgHFzZgAgM1v07F038tAkIBGrYLukY1ZFBaZoGZ9xHfy/EmLJM3HCHLO5 +8wszMGhMTe2+39EeChwgj0kFaq1YnDiucU74BC57KR1tD59y7l6UnsQXTm4/32j8 +Or6i8GekBibCb97DzzOU0ZK//fNhHTXpDDXsYt5lJUWSmgW+S9Qp +-----END RSA PRIVATE KEY----- diff --git a/test/mitmproxy/data/servercert/trusted-leaf.pem b/test/mitmproxy/data/servercert/trusted-leaf.pem new file mode 100644 index 00000000..71700f2a --- /dev/null +++ b/test/mitmproxy/data/servercert/trusted-leaf.pem @@ -0,0 +1,45 @@ +-----BEGIN CERTIFICATE----- +MIIC4TCCAckCCQCj6D9oVylb8jANBgkqhkiG9w0BAQsFADBFMQswCQYDVQQGEwJB +VTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0 +cyBQdHkgTHRkMB4XDTE1MTEwMTE2NDgwMloXDTE4MDgyMTE2NDgwMlowIDEeMBwG +A1UEAwwVZXhhbXBsZS5taXRtcHJveHkub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOC +AQ8AMIIBCgKCAQEAy/L5JYHS7QFhSIsjmd6bJTgs2rdqEn6tsmPBVZKZ7SqCAVjW +hPpEu7Q23akmU6Zm9Fp/vENc3jzxQLlEKhrv7eWmFYSOrCYtbJOz3RQorlwjjfdY +LlNQh1wYUXQX3PN3r3dyYtt5vTtXKc8+aP4M4vX7qlbW+4j4LrQfmPjS0XOdYpu3 +wh+i1ZMIhZye3hpCjwnpjTf7/ff45ZFxtkoi1uzEC/+swr1RSvamY8Foe12Re17Z +5ij8ZB0NIdoSk1tDkY3sJ8iNi35+qartl0UYeG9IUXRwDRrPsEKpF4RxY1+X2bdZ +r6PKb/E4CA5JlMvS5SVmrvxjCVqTQBmTjXfxqwIDAQABMA0GCSqGSIb3DQEBCwUA +A4IBAQBmpSZJrTDvzSlo6P7P7x1LoETzHyVjwgPeqGYw6ndGXeJMN9rhhsFvRsiB +I/aHh58MIlSjti7paikDAoFHB3dBvFHR+JUa/ailWEbcZReWRSE3lV6wFiN3G3lU +OyofR7MKnPW7bv8hSqOLqP1mbupXuQFB5M6vPLRwg5VgiCHI/XBiTvzMamzvNAR3 +UHHZtsJkRqzogYm6K9YJaga7jteSx2nNo+ujLwrxeXsLChTyFMJGnVkp5IyKeNfc +qwlzNncb3y+4KnUdNkPEtuydgAxAfuyXufiFBYRcUWbQ5/9ycgF7131ySaj9f/Y2 +kMsv2jg+soKvwwVYCABsk1KSHtfz +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAy/L5JYHS7QFhSIsjmd6bJTgs2rdqEn6tsmPBVZKZ7SqCAVjW +hPpEu7Q23akmU6Zm9Fp/vENc3jzxQLlEKhrv7eWmFYSOrCYtbJOz3RQorlwjjfdY +LlNQh1wYUXQX3PN3r3dyYtt5vTtXKc8+aP4M4vX7qlbW+4j4LrQfmPjS0XOdYpu3 +wh+i1ZMIhZye3hpCjwnpjTf7/ff45ZFxtkoi1uzEC/+swr1RSvamY8Foe12Re17Z +5ij8ZB0NIdoSk1tDkY3sJ8iNi35+qartl0UYeG9IUXRwDRrPsEKpF4RxY1+X2bdZ +r6PKb/E4CA5JlMvS5SVmrvxjCVqTQBmTjXfxqwIDAQABAoIBAQC956DWq+wbhA1x +3x1nSUBth8E8Z0z9q7dRRFHhvIBXth0X5ADcEa2umj/8ZmSpv2heX2ZRhugSh+yc +t+YgzrRacFwV7ThsU6A4WdBBK2Q19tWke4xAlpOFdtut/Mu7kXkAidiY9ISHD5o5 +9B/I48ZcD3AnTHUiAogV9OL3LbogDD4HasLt4mWkbq8U2thdjxMIvxdg36olJEuo +iAZrAUCPZEXuU89BtvPLUYioe9n90nzkyneGNS0SHxotlEc9ZYK9VTsivtXJb4wB +ptDMCp+TH3tjo8BTGnbnoZEybgyyOEd0UTzxK4DlxnvRVWexFY6NXwPFhIxKlB0Y +Bg8NkAkBAoGBAOiRnmbC5QkqrKrTkLx3fghIHPqgEXPPYgHLSuY3UjTlMb3APXpq +vzQnlCn3QuSse/1fWnQj+9vLVbx1XNgKjzk7dQhn5IUY+mGN4lLmoSnTebxvSQ43 +VAgTYjST9JFmJ3wK4KkWDsEsVao8LAx0h5JEQXUTT5xZpFA2MLztYbgfAoGBAOB/ +MvhLMAwlx8+m/zXMEPLk/KOd2dVZ4q5se8bAT/GiGsi8JUcPnCk140ZZabJqryAp +JFzUHIjfVsS9ejAfocDk1JeIm7Uus4um6fQEKIPMBxI/M/UAwYCXAG9ULXqilbO3 +pTdeeuraVKrTu1Z4ea6x4du1JWKcyDfYfsHepcT1AoGBAM2fskV5G7e3G2MOG3IG +1E/OMpEE5WlXenfLnjVdxDkwS4JRbgnGR7d9JurTyzkTp6ylmfwFtLDoXq15ttTs +wSUBBMCh2tIy+201XV2eu++XIpMQca84C/v352RFTH8hqtdpZqkY74KsCDGzcd6x +SQxxfM5efIzoVPb2crEX0MZRAoGAQ2EqFSfL9flo7UQ8GRN0itJ7mUgJV2WxCZT5 +2X9i/y0eSN1feuKOhjfsTPMNLEWk5kwy48GuBs6xpj8Qa10zGUgVHp4bzdeEgAfK +9DhDSLt1694YZBKkAUpRERj8xXAC6nvWFLZAwjhhbRw7gAqMywgMt/q4i85usYRD +F0ESE/kCgYBbc083PcLmlHbkn/d1i4IcLI6wFk+tZYIEVYDid7xDOgZOBcOTTyYB +BrDzNqbKNexKRt7QHVlwR+VOGMdN5P0hf7oH3SMW23OxBKoQe8pUSGF9a4DjCS1v +vCXMekifb9kIhhUWaG71L8+MaOzNBVAmk1+3NzPZgV/YxHjAWWhGHQ== +-----END RSA PRIVATE KEY----- diff --git a/test/mitmproxy/data/servercert/trusted-root.pem b/test/mitmproxy/data/servercert/trusted-root.pem new file mode 100644 index 00000000..2c75b88e --- /dev/null +++ b/test/mitmproxy/data/servercert/trusted-root.pem @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAPAfPQGCV/Z4MA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTUxMTAxMTY0ODAxWhcNMTgwODIxMTY0ODAxWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEArp8LD34JhKCwcQbwIYQMg4+eCgLVN8fwB7+/qOfJbArPs0djFBN+F7c6 +HGvMr24BKUk5u8pn4dPtNurm/vPC8ovNGmcXz62BQJpcMX2veVdRsF7yNwhNacNJ +Arq+70zNMwYBznx0XUxMF6j6nVFf3AW6SU04ylT4Mp3SY/BUUDAdfl1eRo0mPLNS +8rpsN+8YBw1Q7SCuBRVqpOgVIsL88svgQUSOlzvMZPBpG/cmB3BNKNrltwb5iFEI +1jAV7uSj5IcIuNO/246kfsDVPTFMJIzav/CUoidd5UNw+SoFDlzh8sA7L1Bm7D1/ +3KHYSKswGsSR3kynAl10w/SJKDtn8wIDAQABo1AwTjAdBgNVHQ4EFgQUgOcrtxBX +LxbpnOT65d+vpfyWUkgwHwYDVR0jBBgwFoAUgOcrtxBXLxbpnOT65d+vpfyWUkgw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAEE9bFmUCA+6cvESKPoi2 +TGSpV652d0xd2U66LpEXeiWRJFLz8YGgoJCx3QFGBscJDXxrLxrBBBV/tCpEqypo +pYIqsawH7M66jpOr83Us3M8JC2eFBZJocMpXxdytWqHik5VKZNx6VQFT8bS7+yVC +VoUKePhlgcg+pmo41qjqieBNKRMh/1tXS77DI1lgO5wZLVrLXcdqWuDpmaQOKJeq +G/nxytCW/YJA7bFn/8Gjy8DYypJSeeaKu7o3P3+ONJHdIMHb+MdcheDBS9AOFSeo +xI0D5EbO9F873O77l7nbD7B0X34HFN0nGczC4poexIpbDFG3hAPekwZ5KC6VwJLc +1Q== +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArp8LD34JhKCwcQbwIYQMg4+eCgLVN8fwB7+/qOfJbArPs0dj +FBN+F7c6HGvMr24BKUk5u8pn4dPtNurm/vPC8ovNGmcXz62BQJpcMX2veVdRsF7y +NwhNacNJArq+70zNMwYBznx0XUxMF6j6nVFf3AW6SU04ylT4Mp3SY/BUUDAdfl1e +Ro0mPLNS8rpsN+8YBw1Q7SCuBRVqpOgVIsL88svgQUSOlzvMZPBpG/cmB3BNKNrl +twb5iFEI1jAV7uSj5IcIuNO/246kfsDVPTFMJIzav/CUoidd5UNw+SoFDlzh8sA7 +L1Bm7D1/3KHYSKswGsSR3kynAl10w/SJKDtn8wIDAQABAoIBAFgMzjDzpqz/sbhs +fS0JPp4gDtqRbx3/bSMbJvNuXPxjvzNxLZ5z7cLbmyu1l7Jlz6QXzkrI1vTiPdzR +OcUY+RYANF252iHYJTKEIzS5YX/X7dL3LT9eqlpIJEqCC8Dygw3VW5fY3Xwl+sB7 +blNhMuro4HQRwi8UBUrQlcPa7Ui5BBi323Q6en+VjYctkqpJHzNKPSqPTbsdLaK+ +B0XuXxFatM09rmeRKZCL71Lk1T8N/l0hqEzej7zxgVD7vG/x1kMFN4T3yCmXCbPa +izGHYr1EBHglm4qMNWveXCZiVJ+wmwCjdjqvggyHiZFXE2N0OCrWPhxQPdqFf5y7 +bUO9U2ECgYEA6GM1UzRnbVpjb20ezFy7dU7rlWM0nHBfG27M3bcXh4HnPpnvKp0/ +8a1WFi4kkRywrNXx8hFEd43vTbdObLpVXScXRKiY3MHmFk4k4hbWuTpmumCubQZO +AWlX6TE0HRKn1wQahgpQcxcWaDN2xJJmRQ1zVmlnNkT48/4kFgRxyykCgYEAwF08 +ngrF35oYoU/x+KKq2NXGeNUzoZMj568dE1oWW0ZFpqCi+DGT+hAbG3yUOBSaPqy9 +zn1obGo0YRlrayvtebz118kG7a/rzY02VcAPlT/GpEhvkZlXTwEK17zRJc1nJrfP +39QAZWZsaOru9NRIg/8HcdG3JPR2MhRD/De9GbsCgYAaiZnBUq6s8jGAu/lUZRKT +JtwIRzfu1XZG77Q9bXcmZlM99t41A5gVxTGbftF2MMyMMDJc7lPfQzocqd4u1GiD +Jr+le4tZSls4GNxlZS5IIL8ycW/5y0qFJr5/RrsoxsSb7UAKJothWTWZ2Karc/xx +zkNpjsfWjrHPSypbyU4lYQKBgFh1R5/BgnatjO/5LGNSok/uFkOQfxqo6BTtYOh6 +P9efO/5A1lBdtBeE+oIsSphzWO7DTtE6uB9Kw2V3Y/83hw+5RjABoG8Cu+OdMURD +eqb+WeFH8g45Pn31E8Bbcq34g5u5YR0jhz8Z13ZzuojZabNRPmIntxmGVSf4S78a +/plrAoGBANMHNng2lyr03nqnHrOM6NXD+60af0YR/YJ+2d/H40RnXxGJ4DXn7F00 +a4vJFPa97uq+xpd0HE+TE+NIrOdVDXPePD2qzBzMTsctGtj30vLzojMOT+Yf/nvO +WxTL5Q8GruJz2Dn0awSZO2z/3A8S1rmpuVZ/jT5NtRrvOSY6hmxF +-----END RSA PRIVATE KEY----- diff --git a/test/mitmproxy/data/trusted-cadir/8117bdb9.0 b/test/mitmproxy/data/trusted-cadir/8117bdb9.0 deleted file mode 100644 index ae78b546..00000000 --- a/test/mitmproxy/data/trusted-cadir/8117bdb9.0 +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICJzCCAZACCQCo1BdopddN/TANBgkqhkiG9w0BAQUFADBXMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 -cyBQdHkgTHRkMRAwDgYDVQQDEwdUUlVTVEVEMCAXDTE1MDYxOTE4MDEzMVoYDzIx -MTUwNTI2MTgwMTMxWjBXMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0 -ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAwDgYDVQQDEwdU -UlVTVEVEMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC00Jf3KrBAmLQWl+Dz -8Qrig8ActB94kv0/Lu03P/2DwOR8kH2h3w4OC3b3CFKX31h7hm/H1PPHq7cIX6IR -fwrYCtBE77UbxklSlrwn06j6YSotz0/dwLEQEFDXWITJq7AyntaiafDHazbbXESN -m/+I/YEl2wKemEHE//qWbeM9kwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAF0NREP3 -X+fTebzJGttzrFkDhGVFKRNyLXblXRVanlGOYF+q8grgZY2ufC/55gqf+ub6FRT5 -gKPhL4V2rqL8UAvCE7jq8ujpVfTB8kRAKC675W2DBZk2EJX9mjlr89t7qXGsI5nF -onpfJ1UtiJshNoV7h/NFHeoag91kx628807n ------END CERTIFICATE----- diff --git a/test/mitmproxy/data/trusted-cadir/9d45e6a9.0 b/test/mitmproxy/data/trusted-cadir/9d45e6a9.0 deleted file mode 100644 index ae78b546..00000000 --- a/test/mitmproxy/data/trusted-cadir/9d45e6a9.0 +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICJzCCAZACCQCo1BdopddN/TANBgkqhkiG9w0BAQUFADBXMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 -cyBQdHkgTHRkMRAwDgYDVQQDEwdUUlVTVEVEMCAXDTE1MDYxOTE4MDEzMVoYDzIx -MTUwNTI2MTgwMTMxWjBXMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0 -ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAwDgYDVQQDEwdU -UlVTVEVEMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC00Jf3KrBAmLQWl+Dz -8Qrig8ActB94kv0/Lu03P/2DwOR8kH2h3w4OC3b3CFKX31h7hm/H1PPHq7cIX6IR -fwrYCtBE77UbxklSlrwn06j6YSotz0/dwLEQEFDXWITJq7AyntaiafDHazbbXESN -m/+I/YEl2wKemEHE//qWbeM9kwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAF0NREP3 -X+fTebzJGttzrFkDhGVFKRNyLXblXRVanlGOYF+q8grgZY2ufC/55gqf+ub6FRT5 -gKPhL4V2rqL8UAvCE7jq8ujpVfTB8kRAKC675W2DBZk2EJX9mjlr89t7qXGsI5nF -onpfJ1UtiJshNoV7h/NFHeoag91kx628807n ------END CERTIFICATE----- diff --git a/test/mitmproxy/data/trusted-cadir/trusted-ca.pem b/test/mitmproxy/data/trusted-cadir/trusted-ca.pem deleted file mode 100644 index ae78b546..00000000 --- a/test/mitmproxy/data/trusted-cadir/trusted-ca.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICJzCCAZACCQCo1BdopddN/TANBgkqhkiG9w0BAQUFADBXMQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 -cyBQdHkgTHRkMRAwDgYDVQQDEwdUUlVTVEVEMCAXDTE1MDYxOTE4MDEzMVoYDzIx -MTUwNTI2MTgwMTMxWjBXMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0 -ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAwDgYDVQQDEwdU -UlVTVEVEMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC00Jf3KrBAmLQWl+Dz -8Qrig8ActB94kv0/Lu03P/2DwOR8kH2h3w4OC3b3CFKX31h7hm/H1PPHq7cIX6IR -fwrYCtBE77UbxklSlrwn06j6YSotz0/dwLEQEFDXWITJq7AyntaiafDHazbbXESN -m/+I/YEl2wKemEHE//qWbeM9kwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAF0NREP3 -X+fTebzJGttzrFkDhGVFKRNyLXblXRVanlGOYF+q8grgZY2ufC/55gqf+ub6FRT5 -gKPhL4V2rqL8UAvCE7jq8ujpVfTB8kRAKC675W2DBZk2EJX9mjlr89t7qXGsI5nF -onpfJ1UtiJshNoV7h/NFHeoag91kx628807n ------END CERTIFICATE----- diff --git a/test/mitmproxy/data/trusted-server.crt b/test/mitmproxy/data/trusted-server.crt deleted file mode 100644 index 76f8559a..00000000 --- a/test/mitmproxy/data/trusted-server.crt +++ /dev/null @@ -1,33 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC8jCCAlugAwIBAgICEAcwDQYJKoZIhvcNAQEFBQAwVzELMAkGA1UEBhMCQVUx -EzARBgNVBAgTClNvbWUtU3RhdGUxITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMg -UHR5IEx0ZDEQMA4GA1UEAxMHVFJVU1RFRDAgFw0xNTA2MjAwMTE4MjdaGA8yMTE1 -MDUyNzAxMTgyN1owfjELMAkGA1UEBhMCQVUxEzARBgNVBAgTClNvbWUtU3RhdGUx -ITAfBgNVBAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEUMBIGA1UECxMLSU5U -RVJNIFVOSVQxITAfBgNVBAMTGE9SRyBXSVRIIElOVEVSTUVESUFURSBDQTCBnzAN -BgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAtRPNKgh4WdYGmU2Ae6Tf2Mbd3oaRI/uY -Qm6aKeYk1i7g41C0vVowNcD/qdNpGUNnai/Kak9anHOYyppNo7zHgf3EO8zQ4NTQ -pkDKsdCqbUQcjGfhjWXKnOw+I5er4Rj+MwM1f5cbwb8bYHiSPmXaxzdL0/SNXGAA -ys/UswgwkU8CAwEAAaOBozCBoDAMBgNVHRMEBTADAQH/MB0GA1UdDgQWBBTPkPQW -DAPOIy8mipuEsZcP1694EDBxBgNVHSMEajBooVukWTBXMQswCQYDVQQGEwJBVTET -MBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0cyBQ -dHkgTHRkMRAwDgYDVQQDEwdUUlVTVEVEggkAqNQXaKXXTf0wDQYJKoZIhvcNAQEF -BQADgYEApaPbwonY8l+zSxlY2Fw4WNKfl5nwcTW4fuv/0tZLzvsS6P4hTXxbYJNa -k3hQ1qlrr8DiWJewF85hYvEI2F/7eqS5dhhPTEUFPpsjhbgiqnASvW+WKQIgoY2r -aHgOXi7RNFtTcCgk0UZISWOY7ORLy8Xu6vKrLRjDhyfIbGlqnAs= ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQC1E80qCHhZ1gaZTYB7pN/Yxt3ehpEj+5hCbpop5iTWLuDjULS9 -WjA1wP+p02kZQ2dqL8pqT1qcc5jKmk2jvMeB/cQ7zNDg1NCmQMqx0KptRByMZ+GN -Zcqc7D4jl6vhGP4zAzV/lxvBvxtgeJI+ZdrHN0vT9I1cYADKz9SzCDCRTwIDAQAB -AoGAfKHocKnrzEmXuSSy7meI+vfF9kfA1ndxUSg3S+dwK0uQ1mTSQhI1ZIo2bnlo -uU6/e0Lxm0KLJ2wZGjoifjSNTC8pcxIfAQY4kM9fqoUcXVSBVSS2kByTunhNSVZQ -yQyc+UTq9g1zBnJsZAltn7/PaihU4heWgP/++lposuShqmECQQDaG+7l0qul1xak -9kuZgc88BSTfn9iMK2zIQRcVKuidK4dT3QEp0wmWR5Ue8jq8lvTmVTGNGZbHcheh -KhoZfLgLAkEA1IjwAw/8z02yV3lbc2QUjIl9m9lvjHBoE2sGuSfq/cZskLKrGat+ -CVj3spqVAg22tpQwVBuHiipBziWVnEtiTQJAB9FKfchQSLBt6lm9mfHyKJeSm8VR -8Kw5yO+0URjpn4CI6DOasBIVXOKR8LsD6fCLNJpHHWSWZ+2p9SfaKaGzwwJBAM31 -Scld89qca4fzNZkT0goCrvOZeUy6HVE79Q72zPVSFSD/02kT1BaQ3bB5to5/5aD2 -6AKJjwZoPs7bgykrsD0CQBzU8U/8x2dNQnG0QeqaKQu5kKhZSZ9bsawvrCkxSl6b -WAjl/Jehi5bbQ07zQo3cge6qeR38FCWVCHQ/5wNbc54= ------END RSA PRIVATE KEY----- diff --git a/test/mitmproxy/data/untrusted-server.crt b/test/mitmproxy/data/untrusted-server.crt deleted file mode 100644 index 62e58601..00000000 --- a/test/mitmproxy/data/untrusted-server.crt +++ /dev/null @@ -1,32 +0,0 @@ -# untrusted-interm.crt, self-signed ------BEGIN CERTIFICATE----- -MIICdTCCAd4CCQDRSKOnIMbTgDANBgkqhkiG9w0BAQUFADB+MQswCQYDVQQGEwJB -VTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UEChMYSW50ZXJuZXQgV2lkZ2l0 -cyBQdHkgTHRkMRQwEgYDVQQLEwtJTlRFUk0gVU5JVDEhMB8GA1UEAxMYT1JHIFdJ -VEggSU5URVJNRURJQVRFIENBMCAXDTE1MDYyMDAxMzY0M1oYDzIxMTUwNTI3MDEz -NjQzWjB+MQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UE -ChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRQwEgYDVQQLEwtJTlRFUk0gVU5J -VDEhMB8GA1UEAxMYT1JHIFdJVEggSU5URVJNRURJQVRFIENBMIGfMA0GCSqGSIb3 -DQEBAQUAA4GNADCBiQKBgQC1E80qCHhZ1gaZTYB7pN/Yxt3ehpEj+5hCbpop5iTW -LuDjULS9WjA1wP+p02kZQ2dqL8pqT1qcc5jKmk2jvMeB/cQ7zNDg1NCmQMqx0Kpt -RByMZ+GNZcqc7D4jl6vhGP4zAzV/lxvBvxtgeJI+ZdrHN0vT9I1cYADKz9SzCDCR -TwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAGbObAMEajCz4kj7OP2/DB5SRy2+H/G3 -8Qvc43xlMMNQyYxsDuLOFL0UMRzoKgntrrm2nni8jND+tuMt+hv3ZlBcJlYJ6ynR -sC1ITTC/1SwwwO0AFIyduUEIJYr/B3sgcVYPLcEfeDZgmEQc9Tnc01aEu3lx2+l9 -0JTSPL2L9LdA ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQC1E80qCHhZ1gaZTYB7pN/Yxt3ehpEj+5hCbpop5iTWLuDjULS9 -WjA1wP+p02kZQ2dqL8pqT1qcc5jKmk2jvMeB/cQ7zNDg1NCmQMqx0KptRByMZ+GN -Zcqc7D4jl6vhGP4zAzV/lxvBvxtgeJI+ZdrHN0vT9I1cYADKz9SzCDCRTwIDAQAB -AoGAfKHocKnrzEmXuSSy7meI+vfF9kfA1ndxUSg3S+dwK0uQ1mTSQhI1ZIo2bnlo -uU6/e0Lxm0KLJ2wZGjoifjSNTC8pcxIfAQY4kM9fqoUcXVSBVSS2kByTunhNSVZQ -yQyc+UTq9g1zBnJsZAltn7/PaihU4heWgP/++lposuShqmECQQDaG+7l0qul1xak -9kuZgc88BSTfn9iMK2zIQRcVKuidK4dT3QEp0wmWR5Ue8jq8lvTmVTGNGZbHcheh -KhoZfLgLAkEA1IjwAw/8z02yV3lbc2QUjIl9m9lvjHBoE2sGuSfq/cZskLKrGat+ -CVj3spqVAg22tpQwVBuHiipBziWVnEtiTQJAB9FKfchQSLBt6lm9mfHyKJeSm8VR -8Kw5yO+0URjpn4CI6DOasBIVXOKR8LsD6fCLNJpHHWSWZ+2p9SfaKaGzwwJBAM31 -Scld89qca4fzNZkT0goCrvOZeUy6HVE79Q72zPVSFSD/02kT1BaQ3bB5to5/5aD2 -6AKJjwZoPs7bgykrsD0CQBzU8U/8x2dNQnG0QeqaKQu5kKhZSZ9bsawvrCkxSl6b -WAjl/Jehi5bbQ07zQo3cge6qeR38FCWVCHQ/5wNbc54= ------END RSA PRIVATE KEY----- diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index 8e2042fb..d63ee50e 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -224,7 +224,7 @@ def test_get_content_view(): view_auto.side_effect = ValueError desc, lines, err = cv.get_content_view( - cv.get("JSON"), + cv.get("Auto"), b"[1, 2", ) assert err @@ -282,4 +282,4 @@ def test_pretty_json(): assert cv.pretty_json(b'{"foo": 1}') assert not cv.pretty_json(b"moo") assert cv.pretty_json(b'{"foo" : "\xe4\xb8\x96\xe7\x95\x8c"}') # utf8 with chinese characters - assert not cv.pretty_json(b'{"foo" : "\xFF"}')
\ No newline at end of file + assert not cv.pretty_json(b'{"foo" : "\xFF"}') diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 74992130..d4bf764c 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -504,13 +504,13 @@ class TestState: c = flow.State() f = tutils.tflow() c.add_flow(f) - c.set_limit("~e") + c.set_view_filter("~e") assert not c.view f.error = tutils.terr() assert c.update_flow(f) assert c.view - def test_set_limit(self): + def test_set_view_filter(self): c = flow.State() f = tutils.tflow() @@ -519,24 +519,24 @@ class TestState: c.add_flow(f) assert len(c.view) == 1 - c.set_limit("~s") - assert c.limit_txt == "~s" + c.set_view_filter("~s") + assert c.filter_txt == "~s" assert len(c.view) == 0 f.response = HTTPResponse.wrap(netlib.tutils.tresp()) c.update_flow(f) assert len(c.view) == 1 - c.set_limit(None) + c.set_view_filter(None) assert len(c.view) == 1 f = tutils.tflow() c.add_flow(f) assert len(c.view) == 2 - c.set_limit("~q") + c.set_view_filter("~q") assert len(c.view) == 1 - c.set_limit("~s") + c.set_view_filter("~s") assert len(c.view) == 1 - assert "Invalid" in c.set_limit("~") + assert "Invalid" in c.set_view_filter("~") def test_set_intercept(self): c = flow.State() diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index aa096a72..f0fa9a40 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -102,7 +102,11 @@ class _Http2TestBase(object): @classmethod def get_options(cls): - opts = options.Options(listen_port=0, no_upstream_cert=False) + opts = options.Options( + listen_port=0, + no_upstream_cert=False, + ssl_insecure=True + ) opts.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") return opts diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 6e790e28..84838018 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -146,9 +146,9 @@ class TestProcessProxyOptions: "--singleuser", "test") - def test_verify_upstream_cert(self): - p = self.assert_noerr("--verify-upstream-cert") - assert p.openssl_verification_mode_server == SSL.VERIFY_PEER + def test_insecure(self): + p = self.assert_noerr("--insecure") + assert p.openssl_verification_mode_server == SSL.VERIFY_NONE def test_upstream_trusted_cadir(self): expected_dir = "/path/to/a/ca/dir" diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 6230fc1f..78e9b5c7 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -2,22 +2,21 @@ import os import socket import time import types -from OpenSSL import SSL -from netlib.exceptions import HttpReadDisconnect, HttpException -from netlib.tcp import Address import netlib.tutils +from mitmproxy import controller +from mitmproxy import options +from mitmproxy.builtins import script +from mitmproxy.models import Error, HTTPResponse, HTTPFlow +from mitmproxy.proxy.config import HostMatcher, parse_server_spec from netlib import tcp, http, socks from netlib.certutils import SSLCert +from netlib.exceptions import HttpReadDisconnect, HttpException from netlib.http import authentication, http1 +from netlib.tcp import Address from netlib.tutils import raises from pathod import pathoc, pathod -from mitmproxy.builtins import script -from mitmproxy import controller -from mitmproxy.proxy.config import HostMatcher, parse_server_spec -from mitmproxy.models import Error, HTTPResponse, HTTPFlow - from . import tutils, tservers """ @@ -350,6 +349,15 @@ class TestHTTPSCertfile(tservers.HTTPProxyTest, CommonMixin): assert self.pathod("304") +class TestHTTPSSecureByDefault: + def test_secure_by_default(self): + """ + Certificate verification should be turned on by default. + """ + default_opts = options.Options() + assert not default_opts.ssl_insecure + + class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest): """ @@ -357,26 +365,35 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest): """ ssl = True ssloptions = pathod.SSLOptions( - cn=b"trusted-cert", + cn=b"example.mitmproxy.org", certs=[ - ("trusted-cert", tutils.test_data.path("data/trusted-server.crt")) - ]) + ("example.mitmproxy.org", tutils.test_data.path("data/servercert/trusted-leaf.pem")) + ] + ) + + def _request(self): + p = self.pathoc(sni="example.mitmproxy.org") + return p.request("get:/p/242") def test_verification_w_cadir(self): self.config.options.update( - ssl_verify_upstream_cert = True, - ssl_verify_upstream_trusted_cadir = tutils.test_data.path( - "data/trusted-cadir/" - ) + ssl_insecure=False, + ssl_verify_upstream_trusted_cadir=tutils.test_data.path( + "data/servercert/" + ), + ssl_verify_upstream_trusted_ca=None, ) - self.pathoc() + assert self._request().status_code == 242 def test_verification_w_pemfile(self): - self.config.openssl_verification_mode_server = SSL.VERIFY_PEER - self.config.options.ssl_verify_upstream_trusted_ca = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem") - - self.pathoc() + self.config.options.update( + ssl_insecure=False, + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=tutils.test_data.path( + "data/servercert/trusted-root.pem" + ), + ) + assert self._request().status_code == 242 class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): @@ -386,42 +403,36 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): """ ssl = True ssloptions = pathod.SSLOptions( - cn=b"untrusted-cert", + cn=b"example.mitmproxy.org", certs=[ - ("untrusted-cert", tutils.test_data.path("data/untrusted-server.crt")) + ("example.mitmproxy.org", tutils.test_data.path("data/servercert/self-signed.pem")) ]) def _request(self): - p = self.pathoc() - # We need to make an actual request because the upstream connection is lazy-loaded. + p = self.pathoc(sni="example.mitmproxy.org") return p.request("get:/p/242") - def test_default_verification_w_bad_cert(self): - """Should use no verification.""" - self.config.options.update( - ssl_verify_upstream_trusted_ca = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem" - ) + @classmethod + def get_options(cls): + opts = super(tservers.HTTPProxyTest, cls).get_options() + opts.ssl_verify_upstream_trusted_ca = tutils.test_data.path( + "data/servercert/trusted-root.pem" ) - assert self._request().status_code == 242 + return opts def test_no_verification_w_bad_cert(self): - self.config.options.update( - ssl_verify_upstream_cert = False, - ssl_verify_upstream_trusted_ca = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem" - ) - ) - assert self._request().status_code == 242 + self.config.options.ssl_insecure = True + r = self._request() + assert r.status_code == 242 def test_verification_w_bad_cert(self): - self.config.options.update( - ssl_verify_upstream_cert = True, - ssl_verify_upstream_trusted_ca = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem" - ) - ) - assert self._request().status_code == 502 + # We only test for a single invalid cert here. + # Actual testing of different root-causes (invalid hostname, expired, ...) + # is done in netlib. + self.config.options.ssl_insecure = False + r = self._request() + assert r.status_code == 502 + assert b"Certificate Verification Error" in r.raw_content class TestHTTPSNoCommonName(tservers.HTTPProxyTest): @@ -1021,11 +1032,11 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): class AddUpstreamCertsToClientChainMixin: ssl = True - servercert = tutils.test_data.path("data/trusted-server.crt") + servercert = tutils.test_data.path("data/servercert/trusted-root.pem") ssloptions = pathod.SSLOptions( - cn=b"trusted-cert", + cn=b"example.mitmproxy.org", certs=[ - (b"trusted-cert", servercert) + (b"example.mitmproxy.org", servercert) ] ) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index d364162c..1597f59c 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -120,7 +120,8 @@ class ProxyTestBase(object): return options.Options( listen_port=0, cadir=cls.cadir, - add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain + add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain, + ssl_insecure=True, ) diff --git a/test/netlib/test_encoding.py b/test/netlib/test_encoding.py index a5e81379..08e69ec5 100644 --- a/test/netlib/test_encoding.py +++ b/test/netlib/test_encoding.py @@ -21,6 +21,18 @@ def test_gzip(): encoding.decode(b"bogus", "gzip") +def test_brotli(): + assert b"string" == encoding.decode( + encoding.encode( + b"string", + "br" + ), + "br" + ) + with tutils.raises(ValueError): + encoding.decode(b"bogus", "br") + + def test_deflate(): assert b"string" == encoding.decode( encoding.encode( diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index 7c3eacc6..52299e59 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -48,9 +48,12 @@ def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" - assert strutils.bytes_to_escaped_str(b"'") == r"\'" + assert strutils.bytes_to_escaped_str(b"'") == r"'" assert strutils.bytes_to_escaped_str(b'"') == r'"' + assert strutils.bytes_to_escaped_str(b"'", escape_single_quotes=True) == r"\'" + assert strutils.bytes_to_escaped_str(b'"', escape_single_quotes=True) == r'"' + assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" diff --git a/test/netlib/test_tcp.py b/test/netlib/test_tcp.py index 273427d5..dc2f4e7e 100644 --- a/test/netlib/test_tcp.py +++ b/test/netlib/test_tcp.py @@ -213,7 +213,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): # Verification errors should be saved even if connection isn't aborted # aborted - assert c.ssl_verification_error is not None + assert c.ssl_verification_error testval = b"echo!\n" c.wfile.write(testval) @@ -226,7 +226,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): c.convert_to_ssl(verify_options=SSL.VERIFY_NONE) # Verification errors should be saved even if connection isn't aborted - assert c.ssl_verification_error is not None + assert c.ssl_verification_error testval = b"echo!\n" c.wfile.write(testval) @@ -243,11 +243,11 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) - assert c.ssl_verification_error is not None + assert c.ssl_verification_error # Unknown issuing certificate authority for first certificate - assert c.ssl_verification_error['errno'] == 18 - assert c.ssl_verification_error['depth'] == 0 + assert "errno: 18" in str(c.ssl_verification_error) + assert "depth: 0" in str(c.ssl_verification_error) class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): @@ -276,7 +276,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) - assert c.ssl_verification_error is not None + assert c.ssl_verification_error class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): diff --git a/web/package.json b/web/package.json index 302803f2..fb2c8c30 100644 --- a/web/package.json +++ b/web/package.json @@ -7,6 +7,7 @@ "start": "gulp" }, "jest": { + "testRegex": "__tests__/.*\\Spec.js$", "testPathDirs": [ "<rootDir>/src/js" ], diff --git a/web/src/images/favicon.ico b/web/src/images/favicon.ico Binary files differnew file mode 100644 index 00000000..bfd2fde7 --- /dev/null +++ b/web/src/images/favicon.ico diff --git a/web/src/js/__tests__/ducks/flowView.js b/web/src/js/__tests__/ducks/flowViewSpec.js index d5d9a6d9..d5d9a6d9 100644 --- a/web/src/js/__tests__/ducks/flowView.js +++ b/web/src/js/__tests__/ducks/flowViewSpec.js diff --git a/web/src/js/__tests__/ducks/flows.js b/web/src/js/__tests__/ducks/flowsSpec.js index 2b261cb1..2b261cb1 100644 --- a/web/src/js/__tests__/ducks/flows.js +++ b/web/src/js/__tests__/ducks/flowsSpec.js diff --git a/web/src/js/__tests__/ducks/ui.js b/web/src/js/__tests__/ducks/ui/headerSpec.js index d3242815..8968e636 100644 --- a/web/src/js/__tests__/ducks/ui.js +++ b/web/src/js/__tests__/ducks/ui/headerSpec.js @@ -1,10 +1,10 @@ -jest.unmock('../../ducks/ui') -jest.unmock('../../ducks/flows') +jest.unmock('../../../ducks/ui/header') +jest.unmock('../../../ducks/flows') -import reducer, { setActiveMenu } from '../../ducks/ui' -import * as flowActions from '../../ducks/flows' +import reducer, { setActiveMenu } from '../../../ducks/ui/header' +import * as flowActions from '../../../ducks/flows' -describe('ui reducer', () => { +describe('header reducer', () => { it('should return the initial state', () => { expect(reducer(undefined, {}).activeMenu).toEqual('Start') }) diff --git a/web/src/js/__tests__/ducks/utils/list.js b/web/src/js/__tests__/ducks/utils/listSpec.js index 72d162f2..72d162f2 100644 --- a/web/src/js/__tests__/ducks/utils/list.js +++ b/web/src/js/__tests__/ducks/utils/listSpec.js diff --git a/web/src/js/__tests__/ducks/utils/view.js b/web/src/js/__tests__/ducks/utils/viewSpec.js index f0b147da..af3da173 100644 --- a/web/src/js/__tests__/ducks/utils/view.js +++ b/web/src/js/__tests__/ducks/utils/viewSpec.js @@ -66,11 +66,13 @@ describe('view reduce', () => { it('should update item', () => { const state = createState([ { id: 1, val: 1 }, - { id: 2, val: 2 } + { id: 2, val: 2 }, + { id: 3, val: 3 } ]) const result = createState([ { id: 1, val: 1 }, - { id: 2, val: 3 } + { id: 2, val: 3 }, + { id: 3, val: 3 } ]) expect(reduce(state, view.update({ id: 2, val: 3 }))).toEqual(result) }) diff --git a/web/src/js/ducks/utils/view.js b/web/src/js/ducks/utils/view.js index c00f00bd..6bf0a63e 100755 --- a/web/src/js/ducks/utils/view.js +++ b/web/src/js/ducks/utils/view.js @@ -54,21 +54,29 @@ export default function reduce(state = defaultState, action) { } case UPDATE: - if (state.indexOf[action.item.id] == null) { - return + let hasOldItem = state.indexOf[action.item.id] !== null && state.indexOf[action.item.id] !== undefined + let hasNewItem = action.filter(action.item) + if (!hasNewItem && !hasOldItem) { + return state } - const nextState = { - ...state, - ...sortedRemove(state, action.item.id), + if (hasNewItem && !hasOldItem) { + return { + ...state, + ...sortedInsert(state, action.item, action.sort) + } } - if (!action.filter(action.item)) { - return nextState + if (!hasNewItem && hasOldItem) { + return { + ...state, + ...sortedRemove(state, action.item.id) + } } - return { - ...nextState, - ...sortedInsert(nextState, action.item, action.sort) + if (hasNewItem && hasOldItem) { + return { + ...state, + ...sortedUpdate(state, action.item, action.sort), + } } - case RECEIVE: { const data = action.list.filter(action.filter).sort(action.sort) @@ -110,7 +118,7 @@ export function receive(list, filter = defaultFilter, sort = defaultSort) { function sortedInsert(state, item, sort) { const index = sortedIndex(state.data, item, sort) - const data = [...state.data] + const data = [ ...state.data ] const indexOf = { ...state.indexOf } data.splice(index, 0, item) @@ -134,6 +142,28 @@ function sortedRemove(state, id) { return { data, indexOf } } +function sortedUpdate(state, item, sort) { + let data = [ ...state.data ] + let indexOf = { ...state.indexOf } + let index = indexOf[item.id] + data[index] = item + while (index + 1 < data.length && sort(data[index], data[index + 1]) > 0) { + data[index] = data[index + 1] + data[index + 1] = item + indexOf[item.id] = index + 1 + indexOf[data[index].id] = index + ++index + } + while (index > 0 && sort(data[index], data[index - 1]) < 0) { + data[index] = data[index - 1] + data[index - 1] = item + indexOf[item.id] = index - 1 + indexOf[data[index].id] = index + --index + } + return { data, indexOf } +} + function sortedIndex(list, item, sort) { let low = 0 let high = list.length diff --git a/web/src/templates/index.html b/web/src/templates/index.html index 165d7d3d..db9d2ecb 100644 --- a/web/src/templates/index.html +++ b/web/src/templates/index.html @@ -6,10 +6,11 @@ <meta name="viewport" content="width=device-width, initial-scale=1"> <link rel="stylesheet" href="/static/vendor.css"/> <link rel="stylesheet" href="/static/app.css"/> + <link rel="icon" href="/static/images/favicon.ico" type="image/x-icon"/> <script src="/static/vendor.js"></script> <script src="/static/app.js"></script> </head> <body> <div id="mitmproxy"></div> </body> -</html>
\ No newline at end of file +</html> |