diff options
28 files changed, 238 insertions, 1790 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index dae12978..13782ee8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -5,10 +5,10 @@ environment: CI_DEPS: codecov>=2.0.5 CI_COMMANDS: codecov matrix: - - PYTHON: "C:\\Python27" - TOXENV: "py27" - PYTHON: "C:\\Python35" TOXENV: "py35" + - PYTHON: "C:\\Python27" + TOXENV: "py27" SNAPSHOT_HOST: secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI= @@ -35,8 +35,8 @@ deploy_script: ) { pip install -U virtualenv .\dev.ps1 - cmd /c "python .\release\rtool.py bdist 2>&1" - python .\release\rtool.py upload-snapshot --bdist + cmd /c "python -u .\release\rtool.py bdist 2>&1" + python -u .\release\rtool.py upload-snapshot --bdist --wheel } cache: diff --git a/.travis.yml b/.travis.yml index e9566ebe..a8301ec8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,9 @@ matrix: - os: osx osx_image: xcode7.3 language: generic - env: TOXENV=py35 + env: TOXENV=py35 BDIST=1 - python: 3.5 - env: TOXENV=py35 + env: TOXENV=py35 BDIST=1 - python: 3.5 env: TOXENV=py35 NO_ALPN=1 - python: 2.7 @@ -57,14 +57,14 @@ script: set -o pipefail; python -m tox -- --cov netlib --cov mitmproxy --cov pat after_success: - | - if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] + if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then git fetch --unshallow ./dev.sh 3.5 source venv3.5/bin/activate pip install -e ./release - python ./release/rtool.py bdist - python ./release/rtool.py upload-snapshot --bdist --wheel + python -u ./release/rtool.py bdist + python -u ./release/rtool.py upload-snapshot --bdist fi notifications: diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 9609a421..a08e2eab 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,10 +1,13 @@ - 1813 Aldo Cortesi - 1228 Maximilian Hils - 282 Thomas Kriechbaumer + 2118 Aldo Cortesi + 1666 Maximilian Hils + 450 Thomas Kriechbaumer + 210 Shadab Zafar + 94 Jason 83 Marcelo Glezer + 36 Clemens 28 Jim Shaver 18 Henrik Nordstrom - 17 Shadab Zafar + 16 Matthew Shao 14 David Weinstein 14 Pedro Worcel 13 Thomas Roth @@ -14,39 +17,47 @@ 10 András Veres-Szentkirályi 10 Chris Czub 10 Sandor Nemes + 10 Zohar Lorberbaum 9 Kyle Morton 9 Legend Tang - 9 Matthew Shao 9 Rouli + 9 ikoz 8 Chandler Abraham 8 Jason A. Novak 7 Alexis Hildebrandt 7 Brad Peabody 7 Matthias Urlichs + 7 dufferzafar + 6 Felix Yan 5 Choongwoo Han 5 Sam Cleveland 5 Tomaz Muraus + 5 Will Coster 5 elitest 5 iroiro123 4 Bryan Bishop + 4 Clemens Brunner 4 Marc Liyanage 4 Michael J. Bazzinotti 4 Valtteri Virtanen 4 Wade 524 4 Youhei Sakurai 4 root + 4 yonder 3 Benjamin Lee 3 Chris Neasbitt 3 Eli Shvartsman - 3 Felix Yan 3 Guillem Anguera 3 Kyle Manna 3 MatthewShao 3 Ryan Welton 3 Zack B + 3 redfast00 + 3 requires.io 2 Anant 2 Bennett Blodinger 2 Colin Bendell + 2 Cory Benfield 2 Heikki Hannikainen 2 Israel Nir 2 Jaime Soriano Pastor @@ -59,34 +70,50 @@ 2 Paul 2 Rob Wills 2 Sean Coates + 2 Steven Van Acker 2 Terry Long 2 Wade Catron 2 alts 2 isra17 2 israel - 2 requires.io + 2 jpkrause + 2 lilydjwg + 2 strohu + 2 依云 + 1 Aditya 1 Andrey Plotnikov 1 Andy Smith + 1 Anthony Zhang + 1 BSalita 1 Ben Lerner 1 Bradley Baetz + 1 Brett Randall 1 Chris Hamant + 1 Christian Frichot 1 Dan Wilbraham 1 David Dworken 1 David Shaw + 1 Doug Freed 1 Doug Lethin + 1 Drake Caraker 1 Eric Entzel 1 Felix Wolfsteller 1 FreeArtMan 1 Gabriel Kirkpatrick 1 Henrik Nordström + 1 Israel Blancas 1 Ivaylo Popov 1 JC 1 Jakub Nawalaniec 1 Jakub Wilk 1 James Billingham + 1 Jason Pepas 1 Jean Regisser + 1 Jonathan Jones 1 Jorge Villacorta 1 Kit Randel + 1 Kostya Esmukov + 1 Linmiao Xu 1 Lucas Cimon 1 M. Utku Altinkaya 1 Mathieu Mitchell @@ -98,27 +125,33 @@ 1 Nick Raptis 1 Nicolas Esteves 1 Oleksandr Sheremet + 1 Parth Ganatra 1 Pritam Baral 1 Rich Somerfield 1 Rory McCann 1 Rune Halvorsen 1 Ryo Onodera + 1 Sachin Kelkar 1 Sahn Lam 1 Seppo Yli-Olli 1 Sergey Chipiga 1 Stefan Wärting 1 Steve Phillips - 1 Steven Van Acker + 1 Steven Noble 1 Suyash + 1 Tai Dickerson 1 Tarashish Mishra 1 TearsDontFalls + 1 Thiago Arrais 1 Tim Becker 1 Timothy Elliott 1 Ulrich Petri 1 Vyacheslav Bakhmutov - 1 Will Coster + 1 Wes Turner + 1 Yoginski 1 Yuangxuan Wang 1 capt8bit + 1 cle1000 1 davidpshaw 1 deployable 1 gecko655 @@ -133,4 +166,3 @@ 1 sethp-jive 1 starenka 1 vzvu3k6k - 1 依云 diff --git a/mitmproxy/builtins/script.py b/mitmproxy/builtins/script.py index c960dd1c..ae1d1b91 100644 --- a/mitmproxy/builtins/script.py +++ b/mitmproxy/builtins/script.py @@ -61,13 +61,13 @@ def scriptenv(path, args): try: yield except Exception: - _, _, tb = sys.exc_info() + etype, value, tb = sys.exc_info() scriptdir = os.path.dirname(os.path.abspath(path)) for i, s in enumerate(reversed(traceback.extract_tb(tb))): tb = tb.tb_next if not os.path.abspath(s[0]).startswith(scriptdir): break - ctx.log.error("Script error: %s" % "".join(traceback.format_tb(tb))) + ctx.log.error("Script error: %s" % "".join(traceback.format_exception(etype, value, tb))) finally: sys.argv = oldargs sys.path.pop() diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index c354563f..4cde955a 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -713,6 +713,7 @@ class FlowView(tabs.Tabs): keys = ( ("gzip", "z"), ("deflate", "d"), + ("brotli", "b"), ), callback = self.encode_callback, args = (conn,) @@ -726,6 +727,7 @@ class FlowView(tabs.Tabs): encoding_map = { "z": "gzip", "d": "deflate", + "b": "brotli", } conn.encode(encoding_map[key]) signals.flow_change.send(self, flow = self.flow) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index e155bc01..dacef36d 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -30,8 +30,10 @@ from PIL import ExifTags from PIL import Image from six import BytesIO +import cssutils +import jsbeautifier + from mitmproxy import exceptions -from mitmproxy.contrib import jsbeautifier from mitmproxy.contrib.wbxml import ASCommandResponse from netlib import http from netlib import multidict @@ -44,17 +46,6 @@ try: except ImportError: # pragma no cover pyamf = None -try: - import cssutils -except ImportError: # pragma no cover - cssutils = None -else: - cssutils.log.setLevel(logging.CRITICAL) - - cssutils.ser.prefs.keepComments = True - cssutils.ser.prefs.omitLastSemicolon = False - cssutils.ser.prefs.indentClosingBrace = False - cssutils.ser.prefs.validOnly = False # Default view cutoff *in lines* VIEW_CUTOFF = 512 @@ -271,7 +262,7 @@ class ViewHTMLOutline(View): content_types = ["text/html"] def __call__(self, data, **metadata): - data = data.decode("utf-8") + data = data.decode("utf-8", "replace") h = html2text.HTML2Text(baseurl="") h.ignore_images = True h.body_width = 0 @@ -398,6 +389,7 @@ class ViewJavaScript(View): def __call__(self, data, **metadata): opts = jsbeautifier.default_options() opts.indent_size = 2 + data = data.decode("utf-8", "replace") res = jsbeautifier.beautify(data, opts) return "JavaScript", format_text(res) @@ -410,11 +402,14 @@ class ViewCSS(View): ] def __call__(self, data, **metadata): - if cssutils: - sheet = cssutils.parseString(data) - beautified = sheet.cssText - else: - beautified = data + cssutils.log.setLevel(logging.CRITICAL) + cssutils.ser.prefs.keepComments = True + cssutils.ser.prefs.omitLastSemicolon = False + cssutils.ser.prefs.indentClosingBrace = False + cssutils.ser.prefs.validOnly = False + + sheet = cssutils.parseString(data) + beautified = sheet.cssText return "CSS", format_text(beautified) diff --git a/mitmproxy/contrib/jsbeautifier/__init__.py b/mitmproxy/contrib/jsbeautifier/__init__.py deleted file mode 100644 index e319e8dd..00000000 --- a/mitmproxy/contrib/jsbeautifier/__init__.py +++ /dev/null @@ -1,1153 +0,0 @@ -import sys -import getopt -import re -import string - -# -# Originally written by Einar Lielmanis et al., -# Conversion to python by Einar Lielmanis, einar@jsbeautifier.org, -# MIT licence, enjoy. -# -# Python is not my native language, feel free to push things around. -# -# Use either from command line (script displays its usage when run -# without any parameters), -# -# -# or, alternatively, use it as a module: -# -# import jsbeautifier -# res = jsbeautifier.beautify('your javascript string') -# res = jsbeautifier.beautify_file('some_file.js') -# -# you may specify some options: -# -# opts = jsbeautifier.default_options() -# opts.indent_size = 2 -# res = jsbeautifier.beautify('some javascript', opts) -# -# -# Here are the available options: (read source) - - -class BeautifierOptions: - def __init__(self): - self.indent_size = 4 - self.indent_char = ' ' - self.indent_with_tabs = False - self.preserve_newlines = True - self.max_preserve_newlines = 10. - self.jslint_happy = False - self.brace_style = 'collapse' - self.keep_array_indentation = False - self.keep_function_indentation = False - self.eval_code = False - - - - def __repr__(self): - return \ -"""indent_size = %d -indent_char = [%s] -preserve_newlines = %s -max_preserve_newlines = %d -jslint_happy = %s -indent_with_tabs = %s -brace_style = %s -keep_array_indentation = %s -eval_code = %s -""" % ( self.indent_size, - self.indent_char, - self.preserve_newlines, - self.max_preserve_newlines, - self.jslint_happy, - self.indent_with_tabs, - self.brace_style, - self.keep_array_indentation, - self.eval_code, - ) - - -class BeautifierFlags: - def __init__(self, mode): - self.previous_mode = 'BLOCK' - self.mode = mode - self.var_line = False - self.var_line_tainted = False - self.var_line_reindented = False - self.in_html_comment = False - self.if_line = False - self.in_case = False - self.eat_next_space = False - self.indentation_baseline = -1 - self.indentation_level = 0 - self.ternary_depth = 0 - - -def default_options(): - return BeautifierOptions() - - -def beautify(string, opts = default_options() ): - b = Beautifier() - return b.beautify(string, opts) - -def beautify_file(file_name, opts = default_options() ): - - if file_name == '-': # stdin - f = sys.stdin - else: - try: - f = open(file_name) - except Exception as ex: - return 'The file could not be opened' - - b = Beautifier() - return b.beautify(''.join(f.readlines()), opts) - - -def usage(): - - print("""Javascript beautifier (http://jsbeautifier.org/) - -Usage: jsbeautifier.py [options] <infile> - - <infile> can be "-", which means stdin. - <outfile> defaults to stdout - -Input options: - - -i, --stdin read input from stdin - -Output options: - - -s, --indent-size=NUMBER indentation size. (default 4). - -c, --indent-char=CHAR character to indent with. (default space). - -t, --indent-with-tabs Indent with tabs, overrides -s and -c - -d, --disable-preserve-newlines do not preserve existing line breaks. - -j, --jslint-happy more jslint-compatible output - -b, --brace-style=collapse brace style (collapse, expand, end-expand) - -k, --keep-array-indentation keep array indentation. - -o, --outfile=FILE specify a file to output to (default stdout) - -f, --keep-function-indentation Do not re-indent function bodies defined in var lines. - -Rarely needed options: - - --eval-code evaluate code if a JS interpreter is - installed. May be useful with some obfuscated - script but poses a potential security issue. - - -l, --indent-level=NUMBER initial indentation level. (default 0). - - -h, --help, --usage prints this help statement. - -""") - - - - - - -class Beautifier: - - def __init__(self, opts = default_options() ): - - self.opts = opts - self.blank_state() - - def blank_state(self): - - # internal flags - self.flags = BeautifierFlags('BLOCK') - self.flag_store = [] - self.wanted_newline = False - self.just_added_newline = False - self.do_block_just_closed = False - - if self.opts.indent_with_tabs: - self.indent_string = "\t" - else: - self.indent_string = self.opts.indent_char * self.opts.indent_size - - self.preindent_string = '' - self.last_word = '' # last TK_WORD seen - self.last_type = 'TK_START_EXPR' # last token type - self.last_text = '' # last token text - self.last_last_text = '' # pre-last token text - - self.input = None - self.output = [] # formatted javascript gets built here - - self.whitespace = ["\n", "\r", "\t", " "] - self.wordchar = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$' - self.digits = '0123456789' - self.punct = '+ - * / % & ++ -- = += -= *= /= %= == === != !== > < >= <= >> << >>> >>>= >>= <<= && &= | || ! !! , : ? ^ ^= |= ::' - self.punct += ' <?= <? ?> <%= <% %>' - self.punct = self.punct.split(' ') - - - # Words which always should start on a new line - self.line_starters = 'continue,try,throw,return,var,if,switch,case,default,for,while,break,function'.split(',') - self.set_mode('BLOCK') - - global parser_pos - parser_pos = 0 - - - def beautify(self, s, opts = None ): - - if opts != None: - self.opts = opts - - - if self.opts.brace_style not in ['expand', 'collapse', 'end-expand']: - raise(Exception('opts.brace_style must be "expand", "collapse" or "end-expand".')) - - self.blank_state() - - while s and s[0] in [' ', '\t']: - self.preindent_string += s[0] - s = s[1:] - - #self.input = self.unpack(s, opts.eval_code) - # CORTESI - self.input = s - - parser_pos = 0 - while True: - token_text, token_type = self.get_next_token() - #print (token_text, token_type, self.flags.mode) - if token_type == 'TK_EOF': - break - - handlers = { - 'TK_START_EXPR': self.handle_start_expr, - 'TK_END_EXPR': self.handle_end_expr, - 'TK_START_BLOCK': self.handle_start_block, - 'TK_END_BLOCK': self.handle_end_block, - 'TK_WORD': self.handle_word, - 'TK_SEMICOLON': self.handle_semicolon, - 'TK_STRING': self.handle_string, - 'TK_EQUALS': self.handle_equals, - 'TK_OPERATOR': self.handle_operator, - 'TK_BLOCK_COMMENT': self.handle_block_comment, - 'TK_INLINE_COMMENT': self.handle_inline_comment, - 'TK_COMMENT': self.handle_comment, - 'TK_UNKNOWN': self.handle_unknown, - } - - handlers[token_type](token_text) - - self.last_last_text = self.last_text - self.last_type = token_type - self.last_text = token_text - - sweet_code = self.preindent_string + re.sub('[\n ]+$', '', ''.join(self.output)) - return sweet_code - - def unpack(self, source, evalcode=False): - import jsbeautifier.unpackers as unpackers - try: - return unpackers.run(source, evalcode) - except unpackers.UnpackingError as error: - print('error:', error) - return '' - - def trim_output(self, eat_newlines = False): - while len(self.output) \ - and ( - self.output[-1] == ' '\ - or self.output[-1] == self.indent_string \ - or self.output[-1] == self.preindent_string \ - or (eat_newlines and self.output[-1] in ['\n', '\r'])): - self.output.pop() - - def is_special_word(self, s): - return s in ['case', 'return', 'do', 'if', 'throw', 'else']; - - def is_array(self, mode): - return mode in ['[EXPRESSION]', '[INDENDED-EXPRESSION]'] - - - def is_expression(self, mode): - return mode in ['[EXPRESSION]', '[INDENDED-EXPRESSION]', '(EXPRESSION)', '(FOR-EXPRESSION)', '(COND-EXPRESSION)'] - - - def append_newline_forced(self): - old_array_indentation = self.opts.keep_array_indentation - self.opts.keep_array_indentation = False - self.append_newline() - self.opts.keep_array_indentation = old_array_indentation - - def append_newline(self, ignore_repeated = True): - - self.flags.eat_next_space = False - - if self.opts.keep_array_indentation and self.is_array(self.flags.mode): - return - - self.flags.if_line = False - self.trim_output() - - if len(self.output) == 0: - # no newline on start of file - return - - if self.output[-1] != '\n' or not ignore_repeated: - self.just_added_newline = True - self.output.append('\n') - - if self.preindent_string: - self.output.append(self.preindent_string) - - for i in range(self.flags.indentation_level): - self.output.append(self.indent_string) - - if self.flags.var_line and self.flags.var_line_reindented: - self.output.append(self.indent_string) - - - def append(self, s): - if s == ' ': - # do not add just a single space after the // comment, ever - if self.last_type == 'TK_COMMENT': - return self.append_newline() - - # make sure only single space gets drawn - if self.flags.eat_next_space: - self.flags.eat_next_space = False - elif len(self.output) and self.output[-1] not in [' ', '\n', self.indent_string]: - self.output.append(' ') - else: - self.just_added_newline = False - self.flags.eat_next_space = False - self.output.append(s) - - - def indent(self): - self.flags.indentation_level = self.flags.indentation_level + 1 - - - def remove_indent(self): - if len(self.output) and self.output[-1] in [self.indent_string, self.preindent_string]: - self.output.pop() - - - def set_mode(self, mode): - - prev = BeautifierFlags('BLOCK') - - if self.flags: - self.flag_store.append(self.flags) - prev = self.flags - - self.flags = BeautifierFlags(mode) - - if len(self.flag_store) == 1: - self.flags.indentation_level = 0 - else: - self.flags.indentation_level = prev.indentation_level - if prev.var_line and prev.var_line_reindented: - self.flags.indentation_level = self.flags.indentation_level + 1 - self.flags.previous_mode = prev.mode - - - def restore_mode(self): - self.do_block_just_closed = self.flags.mode == 'DO_BLOCK' - if len(self.flag_store) > 0: - mode = self.flags.mode - self.flags = self.flag_store.pop() - self.flags.previous_mode = mode - - - def get_next_token(self): - - global parser_pos - - self.n_newlines = 0 - - if parser_pos >= len(self.input): - return '', 'TK_EOF' - - self.wanted_newline = False - c = self.input[parser_pos] - parser_pos += 1 - - keep_whitespace = self.opts.keep_array_indentation and self.is_array(self.flags.mode) - - if keep_whitespace: - # slight mess to allow nice preservation of array indentation and reindent that correctly - # first time when we get to the arrays: - # var a = [ - # ....'something' - # we make note of whitespace_count = 4 into flags.indentation_baseline - # so we know that 4 whitespaces in original source match indent_level of reindented source - # - # and afterwards, when we get to - # 'something, - # .......'something else' - # we know that this should be indented to indent_level + (7 - indentation_baseline) spaces - - whitespace_count = 0 - while c in self.whitespace: - if c == '\n': - self.trim_output() - self.output.append('\n') - self.just_added_newline = True - whitespace_count = 0 - elif c == '\t': - whitespace_count += 4 - elif c == '\r': - pass - else: - whitespace_count += 1 - - if parser_pos >= len(self.input): - return '', 'TK_EOF' - - c = self.input[parser_pos] - parser_pos += 1 - - if self.flags.indentation_baseline == -1: - - self.flags.indentation_baseline = whitespace_count - - if self.just_added_newline: - for i in range(self.flags.indentation_level + 1): - self.output.append(self.indent_string) - - if self.flags.indentation_baseline != -1: - for i in range(whitespace_count - self.flags.indentation_baseline): - self.output.append(' ') - - else: # not keep_whitespace - while c in self.whitespace: - if c == '\n': - if self.opts.max_preserve_newlines == 0 or self.opts.max_preserve_newlines > self.n_newlines: - self.n_newlines += 1 - - if parser_pos >= len(self.input): - return '', 'TK_EOF' - - c = self.input[parser_pos] - parser_pos += 1 - - if self.opts.preserve_newlines and self.n_newlines > 1: - for i in range(self.n_newlines): - self.append_newline(i == 0) - self.just_added_newline = True - - self.wanted_newline = self.n_newlines > 0 - - - if c in self.wordchar: - if parser_pos < len(self.input): - while self.input[parser_pos] in self.wordchar: - c = c + self.input[parser_pos] - parser_pos += 1 - if parser_pos == len(self.input): - break - - # small and surprisingly unugly hack for 1E-10 representation - if parser_pos != len(self.input) and self.input[parser_pos] in '+-' \ - and re.match('^[0-9]+[Ee]$', c): - - sign = self.input[parser_pos] - parser_pos += 1 - t = self.get_next_token() - c += sign + t[0] - return c, 'TK_WORD' - - if c == 'in': # in is an operator, need to hack - return c, 'TK_OPERATOR' - - if self.wanted_newline and \ - self.last_type != 'TK_OPERATOR' and\ - self.last_type != 'TK_EQUALS' and\ - not self.flags.if_line and \ - (self.opts.preserve_newlines or self.last_text != 'var'): - self.append_newline() - - return c, 'TK_WORD' - - if c in '([': - return c, 'TK_START_EXPR' - - if c in ')]': - return c, 'TK_END_EXPR' - - if c == '{': - return c, 'TK_START_BLOCK' - - if c == '}': - return c, 'TK_END_BLOCK' - - if c == ';': - return c, 'TK_SEMICOLON' - - if c == '/': - comment = '' - inline_comment = True - comment_mode = 'TK_INLINE_COMMENT' - if self.input[parser_pos] == '*': # peek /* .. */ comment - parser_pos += 1 - if parser_pos < len(self.input): - while not (self.input[parser_pos] == '*' and \ - parser_pos + 1 < len(self.input) and \ - self.input[parser_pos + 1] == '/')\ - and parser_pos < len(self.input): - c = self.input[parser_pos] - comment += c - if c in '\r\n': - comment_mode = 'TK_BLOCK_COMMENT' - parser_pos += 1 - if parser_pos >= len(self.input): - break - parser_pos += 2 - return '/*' + comment + '*/', comment_mode - if self.input[parser_pos] == '/': # peek // comment - comment = c - while self.input[parser_pos] not in '\r\n': - comment += self.input[parser_pos] - parser_pos += 1 - if parser_pos >= len(self.input): - break - parser_pos += 1 - if self.wanted_newline: - self.append_newline() - return comment, 'TK_COMMENT' - - - - if c == "'" or c == '"' or \ - (c == '/' and ((self.last_type == 'TK_WORD' and self.is_special_word(self.last_text)) or \ - (self.last_type == 'TK_END_EXPR' and self.flags.previous_mode in ['(FOR-EXPRESSION)', '(COND-EXPRESSION)']) or \ - (self.last_type in ['TK_COMMENT', 'TK_START_EXPR', 'TK_START_BLOCK', 'TK_END_BLOCK', 'TK_OPERATOR', - 'TK_EQUALS', 'TK_EOF', 'TK_SEMICOLON']))): - sep = c - esc = False - resulting_string = c - in_char_class = False - - if parser_pos < len(self.input): - if sep == '/': - # handle regexp - in_char_class = False - while esc or in_char_class or self.input[parser_pos] != sep: - resulting_string += self.input[parser_pos] - if not esc: - esc = self.input[parser_pos] == '\\' - if self.input[parser_pos] == '[': - in_char_class = True - elif self.input[parser_pos] == ']': - in_char_class = False - else: - esc = False - parser_pos += 1 - if parser_pos >= len(self.input): - # incomplete regex when end-of-file reached - # bail out with what has received so far - return resulting_string, 'TK_STRING' - else: - # handle string - while esc or self.input[parser_pos] != sep: - resulting_string += self.input[parser_pos] - if not esc: - esc = self.input[parser_pos] == '\\' - else: - esc = False - parser_pos += 1 - if parser_pos >= len(self.input): - # incomplete string when end-of-file reached - # bail out with what has received so far - return resulting_string, 'TK_STRING' - - - parser_pos += 1 - resulting_string += sep - if sep == '/': - # regexps may have modifiers /regexp/MOD, so fetch those too - while parser_pos < len(self.input) and self.input[parser_pos] in self.wordchar: - resulting_string += self.input[parser_pos] - parser_pos += 1 - return resulting_string, 'TK_STRING' - - if c == '#': - - # she-bang - if len(self.output) == 0 and len(self.input) > 1 and self.input[parser_pos] == '!': - resulting_string = c - while parser_pos < len(self.input) and c != '\n': - c = self.input[parser_pos] - resulting_string += c - parser_pos += 1 - self.output.append(resulting_string.strip() + "\n") - self.append_newline() - return self.get_next_token() - - - # Spidermonkey-specific sharp variables for circular references - # https://developer.mozilla.org/En/Sharp_variables_in_JavaScript - # http://mxr.mozilla.org/mozilla-central/source/js/src/jsscan.cpp around line 1935 - sharp = '#' - if parser_pos < len(self.input) and self.input[parser_pos] in self.digits: - while True: - c = self.input[parser_pos] - sharp += c - parser_pos += 1 - if parser_pos >= len(self.input) or c == '#' or c == '=': - break - if c == '#' or parser_pos >= len(self.input): - pass - elif self.input[parser_pos] == '[' and self.input[parser_pos + 1] == ']': - sharp += '[]' - parser_pos += 2 - elif self.input[parser_pos] == '{' and self.input[parser_pos + 1] == '}': - sharp += '{}' - parser_pos += 2 - return sharp, 'TK_WORD' - - if c == '<' and self.input[parser_pos - 1 : parser_pos + 3] == '<!--': - parser_pos += 3 - c = '<!--' - while parser_pos < len(self.input) and self.input[parser_pos] != '\n': - c += self.input[parser_pos] - parser_pos += 1 - self.flags.in_html_comment = True - return c, 'TK_COMMENT' - - if c == '-' and self.flags.in_html_comment and self.input[parser_pos - 1 : parser_pos + 2] == '-->': - self.flags.in_html_comment = False - parser_pos += 2 - if self.wanted_newline: - self.append_newline() - return '-->', 'TK_COMMENT' - - if c in self.punct: - while parser_pos < len(self.input) and c + self.input[parser_pos] in self.punct: - c += self.input[parser_pos] - parser_pos += 1 - if parser_pos >= len(self.input): - break - if c == '=': - return c, 'TK_EQUALS' - else: - return c, 'TK_OPERATOR' - return c, 'TK_UNKNOWN' - - - - def handle_start_expr(self, token_text): - if token_text == '[': - if self.last_type == 'TK_WORD' or self.last_text == ')': - if self.last_text in self.line_starters: - self.append(' ') - self.set_mode('(EXPRESSION)') - self.append(token_text) - return - - if self.flags.mode in ['[EXPRESSION]', '[INDENTED-EXPRESSION]']: - if self.last_last_text == ']' and self.last_text == ',': - # ], [ goes to a new line - if self.flags.mode == '[EXPRESSION]': - self.flags.mode = '[INDENTED-EXPRESSION]' - if not self.opts.keep_array_indentation: - self.indent() - self.set_mode('[EXPRESSION]') - if not self.opts.keep_array_indentation: - self.append_newline() - elif self.last_text == '[': - if self.flags.mode == '[EXPRESSION]': - self.flags.mode = '[INDENTED-EXPRESSION]' - if not self.opts.keep_array_indentation: - self.indent() - self.set_mode('[EXPRESSION]') - - if not self.opts.keep_array_indentation: - self.append_newline() - else: - self.set_mode('[EXPRESSION]') - else: - self.set_mode('[EXPRESSION]') - else: - if self.last_text == 'for': - self.set_mode('(FOR-EXPRESSION)') - elif self.last_text in ['if', 'while']: - self.set_mode('(COND-EXPRESSION)') - else: - self.set_mode('(EXPRESSION)') - - - if self.last_text == ';' or self.last_type == 'TK_START_BLOCK': - self.append_newline() - elif self.last_type in ['TK_END_EXPR', 'TK_START_EXPR', 'TK_END_BLOCK'] or self.last_text == '.': - # do nothing on (( and )( and ][ and ]( and .( - if self.wanted_newline: - self.append_newline(); - elif self.last_type not in ['TK_WORD', 'TK_OPERATOR']: - self.append(' ') - elif self.last_word == 'function' or self.last_word == 'typeof': - # function() vs function (), typeof() vs typeof () - if self.opts.jslint_happy: - self.append(' ') - elif self.last_text in self.line_starters or self.last_text == 'catch': - self.append(' ') - - self.append(token_text) - - - def handle_end_expr(self, token_text): - if token_text == ']': - if self.opts.keep_array_indentation: - if self.last_text == '}': - self.remove_indent() - self.append(token_text) - self.restore_mode() - return - else: - if self.flags.mode == '[INDENTED-EXPRESSION]': - if self.last_text == ']': - self.restore_mode() - self.append_newline() - self.append(token_text) - return - self.restore_mode() - self.append(token_text) - - - def handle_start_block(self, token_text): - if self.last_word == 'do': - self.set_mode('DO_BLOCK') - else: - self.set_mode('BLOCK') - - if self.opts.brace_style == 'expand': - if self.last_type != 'TK_OPERATOR': - if self.last_text == '=' or (self.is_special_word(self.last_text) and self.last_text != 'else'): - self.append(' ') - else: - self.append_newline(True) - - self.append(token_text) - self.indent() - else: - if self.last_type not in ['TK_OPERATOR', 'TK_START_EXPR']: - if self.last_type == 'TK_START_BLOCK': - self.append_newline() - else: - self.append(' ') - else: - # if TK_OPERATOR or TK_START_EXPR - if self.is_array(self.flags.previous_mode) and self.last_text == ',': - if self.last_last_text == '}': - self.append(' ') - else: - self.append_newline() - self.indent() - self.append(token_text) - - - def handle_end_block(self, token_text): - self.restore_mode() - if self.opts.brace_style == 'expand': - if self.last_text != '{': - self.append_newline() - else: - if self.last_type == 'TK_START_BLOCK': - if self.just_added_newline: - self.remove_indent() - else: - # {} - self.trim_output() - else: - if self.is_array(self.flags.mode) and self.opts.keep_array_indentation: - self.opts.keep_array_indentation = False - self.append_newline() - self.opts.keep_array_indentation = True - else: - self.append_newline() - - self.append(token_text) - - - def handle_word(self, token_text): - if self.do_block_just_closed: - self.append(' ') - self.append(token_text) - self.append(' ') - self.do_block_just_closed = False - return - - if token_text == 'function': - - if self.flags.var_line: - self.flags.var_line_reindented = not self.opts.keep_function_indentation - if (self.just_added_newline or self.last_text == ';') and self.last_text != '{': - # make sure there is a nice clean space of at least one blank line - # before a new function definition - have_newlines = self.n_newlines - if not self.just_added_newline: - have_newlines = 0 - if not self.opts.preserve_newlines: - have_newlines = 1 - for i in range(2 - have_newlines): - self.append_newline(False) - - if token_text in ['case', 'default']: - if self.last_text == ':': - self.remove_indent() - else: - self.flags.indentation_level -= 1 - self.append_newline() - self.flags.indentation_level += 1 - self.append(token_text) - self.flags.in_case = True - return - - prefix = 'NONE' - - if self.last_type == 'TK_END_BLOCK': - if token_text not in ['else', 'catch', 'finally']: - prefix = 'NEWLINE' - else: - if self.opts.brace_style in ['expand', 'end-expand']: - prefix = 'NEWLINE' - else: - prefix = 'SPACE' - self.append(' ') - elif self.last_type == 'TK_SEMICOLON' and self.flags.mode in ['BLOCK', 'DO_BLOCK']: - prefix = 'NEWLINE' - elif self.last_type == 'TK_SEMICOLON' and self.is_expression(self.flags.mode): - prefix = 'SPACE' - elif self.last_type == 'TK_STRING': - prefix = 'NEWLINE' - elif self.last_type == 'TK_WORD': - if self.last_text == 'else': - # eat newlines between ...else *** some_op... - # won't preserve extra newlines in this place (if any), but don't care that much - self.trim_output(True) - prefix = 'SPACE' - elif self.last_type == 'TK_START_BLOCK': - prefix = 'NEWLINE' - elif self.last_type == 'TK_END_EXPR': - self.append(' ') - prefix = 'NEWLINE' - - if self.flags.if_line and self.last_type == 'TK_END_EXPR': - self.flags.if_line = False - - if token_text in self.line_starters: - if self.last_text == 'else': - prefix = 'SPACE' - else: - prefix = 'NEWLINE' - - if token_text == 'function' and self.last_text in ['get', 'set']: - prefix = 'SPACE' - - if token_text in ['else', 'catch', 'finally']: - if self.last_type != 'TK_END_BLOCK' \ - or self.opts.brace_style == 'expand' \ - or self.opts.brace_style == 'end-expand': - self.append_newline() - else: - self.trim_output(True) - self.append(' ') - elif prefix == 'NEWLINE': - if token_text == 'function' and (self.last_type == 'TK_START_EXPR' or self.last_text in '=,'): - # no need to force newline on "function" - - # (function... - pass - elif token_text == 'function' and self.last_text == 'new': - self.append(' ') - elif self.is_special_word(self.last_text): - # no newline between return nnn - self.append(' ') - elif self.last_type != 'TK_END_EXPR': - if (self.last_type != 'TK_START_EXPR' or token_text != 'var') and self.last_text != ':': - # no need to force newline on VAR - - # for (var x = 0... - if token_text == 'if' and self.last_word == 'else' and self.last_text != '{': - self.append(' ') - else: - self.flags.var_line = False - self.flags.var_line_reindented = False - self.append_newline() - elif token_text in self.line_starters and self.last_text != ')': - self.flags.var_line = False - self.flags.var_line_reindented = False - self.append_newline() - elif self.is_array(self.flags.mode) and self.last_text == ',' and self.last_last_text == '}': - self.append_newline() # }, in lists get a newline - elif prefix == 'SPACE': - self.append(' ') - - - self.append(token_text) - self.last_word = token_text - - if token_text == 'var': - self.flags.var_line = True - self.flags.var_line_reindented = False - self.flags.var_line_tainted = False - - - if token_text == 'if': - self.flags.if_line = True - - if token_text == 'else': - self.flags.if_line = False - - - def handle_semicolon(self, token_text): - self.append(token_text) - self.flags.var_line = False - self.flags.var_line_reindented = False - if self.flags.mode == 'OBJECT': - # OBJECT mode is weird and doesn't get reset too well. - self.flags.mode = 'BLOCK' - - - def handle_string(self, token_text): - if self.last_type == 'TK_END_EXPR' and self.flags.previous_mode in ['(COND-EXPRESSION)', '(FOR-EXPRESSION)']: - self.append(' ') - if self.last_type in ['TK_STRING', 'TK_START_BLOCK', 'TK_END_BLOCK', 'TK_SEMICOLON']: - self.append_newline() - elif self.last_type == 'TK_WORD': - self.append(' ') - - # Try to replace readable \x-encoded characters with their equivalent, - # if it is possible (e.g. '\x41\x42\x43\x01' becomes 'ABC\x01'). - def unescape(match): - block, code = match.group(0, 1) - char = chr(int(code, 16)) - if block.count('\\') == 1 and char in string.printable: - return char - return block - - token_text = re.sub(r'\\{1,2}x([a-fA-F0-9]{2})', unescape, token_text) - - self.append(token_text) - - def handle_equals(self, token_text): - if self.flags.var_line: - # just got an '=' in a var-line, different line breaking rules will apply - self.flags.var_line_tainted = True - - self.append(' ') - self.append(token_text) - self.append(' ') - - - def handle_operator(self, token_text): - space_before = True - space_after = True - - if self.flags.var_line and token_text == ',' and self.is_expression(self.flags.mode): - # do not break on comma, for ( var a = 1, b = 2 - self.flags.var_line_tainted = False - - if self.flags.var_line and token_text == ',': - if self.flags.var_line_tainted: - self.append(token_text) - self.flags.var_line_reindented = True - self.flags.var_line_tainted = False - self.append_newline() - return - else: - self.flags.var_line_tainted = False - - if self.is_special_word(self.last_text): - # return had a special handling in TK_WORD - self.append(' ') - self.append(token_text) - return - - if token_text == ':' and self.flags.in_case: - self.append(token_text) - self.append_newline() - self.flags.in_case = False - return - - if token_text == '::': - # no spaces around the exotic namespacing syntax operator - self.append(token_text) - return - - if token_text == ',': - if self.flags.var_line: - if self.flags.var_line_tainted: - # This never happens, as it's handled previously, right? - self.append(token_text) - self.append_newline() - self.flags.var_line_tainted = False - else: - self.append(token_text) - self.append(' ') - elif self.last_type == 'TK_END_BLOCK' and self.flags.mode != '(EXPRESSION)': - self.append(token_text) - if self.flags.mode == 'OBJECT' and self.last_text == '}': - self.append_newline() - else: - self.append(' ') - else: - if self.flags.mode == 'OBJECT': - self.append(token_text) - self.append_newline() - else: - # EXPR or DO_BLOCK - self.append(token_text) - self.append(' ') - # comma handled - return - elif token_text in ['--', '++', '!'] \ - or (token_text in ['+', '-'] \ - and self.last_type in ['TK_START_BLOCK', 'TK_START_EXPR', 'TK_EQUALS', 'TK_OPERATOR']) \ - or self.last_text in self.line_starters: - - space_before = False - space_after = False - - if self.last_text == ';' and self.is_expression(self.flags.mode): - # for (;; ++i) - # ^^ - space_before = True - - if self.last_type == 'TK_WORD' and self.last_text in self.line_starters: - space_before = True - - if self.flags.mode == 'BLOCK' and self.last_text in ['{', ';']: - # { foo: --i } - # foo(): --bar - self.append_newline() - - elif token_text == '.': - # decimal digits or object.property - space_before = False - - elif token_text == ':': - if self.flags.ternary_depth == 0: - self.flags.mode = 'OBJECT' - space_before = False - else: - self.flags.ternary_depth -= 1 - elif token_text == '?': - self.flags.ternary_depth += 1 - - if space_before: - self.append(' ') - - self.append(token_text) - - if space_after: - self.append(' ') - - - - def handle_block_comment(self, token_text): - - lines = token_text.replace('\x0d', '').split('\x0a') - # all lines start with an asterisk? that's a proper box comment - if not any(l for l in lines[1:] if ( l.strip() == '' or (l.lstrip())[0] != '*')): - self.append_newline() - self.append(lines[0]) - for line in lines[1:]: - self.append_newline() - self.append(' ' + line.strip()) - else: - # simple block comment: leave intact - if len(lines) > 1: - # multiline comment starts on a new line - self.append_newline() - else: - # single line /* ... */ comment stays on the same line - self.append(' ') - for line in lines: - self.append(line) - self.append('\n') - self.append_newline() - - - def handle_inline_comment(self, token_text): - self.append(' ') - self.append(token_text) - if self.is_expression(self.flags.mode): - self.append(' ') - else: - self.append_newline_forced() - - - def handle_comment(self, token_text): - if self.wanted_newline: - self.append_newline() - else: - self.append(' ') - - self.append(token_text) - self.append_newline_forced() - - - def handle_unknown(self, token_text): - if self.last_text in ['return', 'throw']: - self.append(' ') - - self.append(token_text) - - - - - -def main(): - - argv = sys.argv[1:] - - try: - opts, args = getopt.getopt(argv, "s:c:o:djbkil:htf", ['indent-size=','indent-char=','outfile=', 'disable-preserve-newlines', - 'jslint-happy', 'brace-style=', - 'keep-array-indentation', 'indent-level=', 'help', - 'usage', 'stdin', 'eval-code', 'indent-with-tabs', 'keep-function-indentation']) - except getopt.GetoptError: - return usage() - - js_options = default_options() - - file = None - outfile = 'stdout' - if len(args) == 1: - file = args[0] - - for opt, arg in opts: - if opt in ('--keep-array-indentation', '-k'): - js_options.keep_array_indentation = True - if opt in ('--keep-function-indentation','-f'): - js_options.keep_function_indentation = True - elif opt in ('--outfile', '-o'): - outfile = arg - elif opt in ('--indent-size', '-s'): - js_options.indent_size = int(arg) - elif opt in ('--indent-char', '-c'): - js_options.indent_char = arg - elif opt in ('--indent-with-tabs', '-t'): - js_options.indent_with_tabs = True - elif opt in ('--disable-preserve_newlines', '-d'): - js_options.preserve_newlines = False - elif opt in ('--jslint-happy', '-j'): - js_options.jslint_happy = True - elif opt in ('--eval-code'): - js_options.eval_code = True - elif opt in ('--brace-style', '-b'): - js_options.brace_style = arg - elif opt in ('--stdin', '-i'): - file = '-' - elif opt in ('--help', '--usage', '-h'): - return usage() - - if not file: - return usage() - else: - if outfile == 'stdout': - print(beautify_file(file, js_options)) - else: - with open(outfile, 'w') as f: - f.write(beautify_file(file, js_options) + '\n') - diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/README.specs.mkd b/mitmproxy/contrib/jsbeautifier/unpackers/README.specs.mkd deleted file mode 100644 index e937b762..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/README.specs.mkd +++ /dev/null @@ -1,25 +0,0 @@ -# UNPACKERS SPECIFICATIONS - -Nothing very difficult: an unpacker is a submodule placed in the directory -where this file was found. Each unpacker must define three symbols: - - * `PRIORITY` : integer number expressing the priority in applying this - unpacker. Lower number means higher priority. - Makes sense only if a source file has been packed with - more than one packer. - * `detect(source)` : returns `True` if source is packed, otherwise, `False`. - * `unpack(source)` : takes a `source` string and unpacks it. Must always return - valid JavaScript. That is to say, your code should look - like: - -``` -if detect(source): - return do_your_fancy_things_with(source) -else: - return source -``` - -*You can safely define any other symbol in your module, as it will be ignored.* - -`__init__` code will automatically load new unpackers, without any further step -to be accomplished. Simply drop it in this directory. diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/__init__.py b/mitmproxy/contrib/jsbeautifier/unpackers/__init__.py deleted file mode 100644 index fcb5b07a..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/__init__.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# General code for JSBeautifier unpackers infrastructure. See README.specs -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# - -"""General code for JSBeautifier unpackers infrastructure.""" - -import pkgutil -import re -from . import evalbased - -# NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js! -BLACKLIST = ['jsbeautifier.unpackers.evalbased'] - -class UnpackingError(Exception): - """Badly packed source or general error. Argument is a - meaningful description.""" - -def getunpackers(): - """Scans the unpackers dir, finds unpackers and add them to UNPACKERS list. - An unpacker will be loaded only if it is a valid python module (name must - adhere to naming conventions) and it is not blacklisted (i.e. inserted - into BLACKLIST.""" - path = __path__ - prefix = __name__ + '.' - unpackers = [] - interface = ['unpack', 'detect', 'PRIORITY'] - for _importer, modname, _ispkg in pkgutil.iter_modules(path, prefix): - if 'tests' not in modname and modname not in BLACKLIST: - try: - module = __import__(modname, fromlist=interface) - except ImportError: - raise UnpackingError('Bad unpacker: %s' % modname) - else: - unpackers.append(module) - - return sorted(unpackers, key = lambda mod: mod.PRIORITY) - -UNPACKERS = getunpackers() - -def run(source, evalcode=False): - """Runs the applicable unpackers and return unpacked source as a string.""" - for unpacker in [mod for mod in UNPACKERS if mod.detect(source)]: - source = unpacker.unpack(source) - if evalcode and evalbased.detect(source): - source = evalbased.unpack(source) - return source - -def filtercomments(source): - """NOT USED: strips trailing comments and put them at the top.""" - trailing_comments = [] - comment = True - - while comment: - if re.search(r'^\s*\/\*', source): - comment = source[0, source.index('*/') + 2] - elif re.search(r'^\s*\/\/', source): - comment = re.search(r'^\s*\/\/', source).group(0) - else: - comment = None - - if comment: - source = re.sub(r'^\s+', '', source[len(comment):]) - trailing_comments.append(comment) - - return '\n'.join(trailing_comments) + source diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/evalbased.py b/mitmproxy/contrib/jsbeautifier/unpackers/evalbased.py deleted file mode 100644 index b17d926e..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/evalbased.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Unpacker for eval() based packers, a part of javascript beautifier -# by Einar Lielmanis <einar@jsbeautifier.org> -# -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# usage: -# -# if detect(some_string): -# unpacked = unpack(some_string) -# - -"""Unpacker for eval() based packers: runs JS code and returns result. -Works only if a JS interpreter (e.g. Mozilla's Rhino) is installed and -properly set up on host.""" - -from subprocess import PIPE, Popen - -PRIORITY = 3 - -def detect(source): - """Detects if source is likely to be eval() packed.""" - return source.strip().lower().startswith('eval(function(') - -def unpack(source): - """Runs source and return resulting code.""" - return jseval('print %s;' % source[4:]) if detect(source) else source - -# In case of failure, we'll just return the original, without crashing on user. -def jseval(script): - """Run code in the JS interpreter and return output.""" - try: - interpreter = Popen(['js'], stdin=PIPE, stdout=PIPE) - except OSError: - return script - result, errors = interpreter.communicate(script) - if interpreter.poll() or errors: - return script - return result diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/javascriptobfuscator.py b/mitmproxy/contrib/jsbeautifier/unpackers/javascriptobfuscator.py deleted file mode 100644 index aa4344a3..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/javascriptobfuscator.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# simple unpacker/deobfuscator for scripts messed up with -# javascriptobfuscator.com -# -# written by Einar Lielmanis <einar@jsbeautifier.org> -# rewritten in Python by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# Will always return valid javascript: if `detect()` is false, `code` is -# returned, unmodified. -# -# usage: -# -# if javascriptobfuscator.detect(some_string): -# some_string = javascriptobfuscator.unpack(some_string) -# - -"""deobfuscator for scripts messed up with JavascriptObfuscator.com""" - -import re - -PRIORITY = 1 - -def smartsplit(code): - """Split `code` at " symbol, only if it is not escaped.""" - strings = [] - pos = 0 - while pos < len(code): - if code[pos] == '"': - word = '' # new word - pos += 1 - while pos < len(code): - if code[pos] == '"': - break - if code[pos] == '\\': - word += '\\' - pos += 1 - word += code[pos] - pos += 1 - strings.append('"%s"' % word) - pos += 1 - return strings - -def detect(code): - """Detects if `code` is JavascriptObfuscator.com packed.""" - # prefer `is not` idiom, so that a true boolean is returned - return (re.search(r'^var _0x[a-f0-9]+ ?\= ?\[', code) is not None) - -def unpack(code): - """Unpacks JavascriptObfuscator.com packed code.""" - if detect(code): - matches = re.search(r'var (_0x[a-f\d]+) ?\= ?\[(.*?)\];', code) - if matches: - variable = matches.group(1) - dictionary = smartsplit(matches.group(2)) - code = code[len(matches.group(0)):] - for key, value in enumerate(dictionary): - code = code.replace(r'%s[%s]' % (variable, key), value) - return code diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py b/mitmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py deleted file mode 100644 index 9893f95f..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/myobfuscate.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# deobfuscator for scripts messed up with myobfuscate.com -# by Einar Lielmanis <einar@jsbeautifier.org> -# -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# usage: -# -# if detect(some_string): -# unpacked = unpack(some_string) -# - -# CAVEAT by Einar Lielmanis - -# -# You really don't want to obfuscate your scripts there: they're tracking -# your unpackings, your script gets turned into something like this, -# as of 2011-08-26: -# -# var _escape = 'your_script_escaped'; -# var _111 = document.createElement('script'); -# _111.src = 'http://api.www.myobfuscate.com/?getsrc=ok' + -# '&ref=' + encodeURIComponent(document.referrer) + -# '&url=' + encodeURIComponent(document.URL); -# var 000 = document.getElementsByTagName('head')[0]; -# 000.appendChild(_111); -# document.write(unescape(_escape)); -# - -"""Deobfuscator for scripts messed up with MyObfuscate.com""" - -import re -import base64 - -# Python 2 retrocompatibility -# pylint: disable=F0401 -# pylint: disable=E0611 -try: - from urllib import unquote -except ImportError: - from urllib.parse import unquote - -from . import UnpackingError - -PRIORITY = 1 - -CAVEAT = """// -// Unpacker warning: be careful when using myobfuscate.com for your projects: -// scripts obfuscated by the free online version call back home. -// - -""" - -SIGNATURE = (r'["\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F' - r'\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65' - r'\x66\x67\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75' - r'\x76\x77\x78\x79\x7A\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x2B' - r'\x2F\x3D","","\x63\x68\x61\x72\x41\x74","\x69\x6E\x64\x65\x78' - r'\x4F\x66","\x66\x72\x6F\x6D\x43\x68\x61\x72\x43\x6F\x64\x65","' - r'\x6C\x65\x6E\x67\x74\x68"]') - -def detect(source): - """Detects MyObfuscate.com packer.""" - return SIGNATURE in source - -def unpack(source): - """Unpacks js code packed with MyObfuscate.com""" - if not detect(source): - return source - payload = unquote(_filter(source)) - match = re.search(r"^var _escape\='<script>(.*)<\/script>'", - payload, re.DOTALL) - polished = match.group(1) if match else source - return CAVEAT + polished - -def _filter(source): - """Extracts and decode payload (original file) from `source`""" - try: - varname = re.search(r'eval\(\w+\(\w+\((\w+)\)\)\);', source).group(1) - reverse = re.search(r"var +%s *\= *'(.*)';" % varname, source).group(1) - except AttributeError: - raise UnpackingError('Malformed MyObfuscate data.') - try: - return base64.b64decode(reverse[::-1].encode('utf8')).decode('utf8') - except TypeError: - raise UnpackingError('MyObfuscate payload is not base64-encoded.') diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/packer.py b/mitmproxy/contrib/jsbeautifier/unpackers/packer.py deleted file mode 100644 index 4ada669e..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/packer.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Unpacker for Dean Edward's p.a.c.k.e.r, a part of javascript beautifier -# by Einar Lielmanis <einar@jsbeautifier.org> -# -# written by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# usage: -# -# if detect(some_string): -# unpacked = unpack(some_string) -# - -"""Unpacker for Dean Edward's p.a.c.k.e.r""" - -import re -from . import UnpackingError - -PRIORITY = 1 - -def detect(source): - """Detects whether `source` is P.A.C.K.E.R. coded.""" - return source.replace(' ', '').startswith('eval(function(p,a,c,k,e,r') - -def unpack(source): - """Unpacks P.A.C.K.E.R. packed js code.""" - payload, symtab, radix, count = _filterargs(source) - - if count != len(symtab): - raise UnpackingError('Malformed p.a.c.k.e.r. symtab.') - - try: - unbase = Unbaser(radix) - except TypeError: - raise UnpackingError('Unknown p.a.c.k.e.r. encoding.') - - def lookup(match): - """Look up symbols in the synthetic symtab.""" - word = match.group(0) - return symtab[unbase(word)] or word - - source = re.sub(r'\b\w+\b', lookup, payload) - return _replacestrings(source) - -def _filterargs(source): - """Juice from a source file the four args needed by decoder.""" - argsregex = (r"}\('(.*)', *(\d+), *(\d+), *'(.*)'\." - r"split\('\|'\), *(\d+), *(.*)\)\)") - args = re.search(argsregex, source, re.DOTALL).groups() - - try: - return args[0], args[3].split('|'), int(args[1]), int(args[2]) - except ValueError: - raise UnpackingError('Corrupted p.a.c.k.e.r. data.') - -def _replacestrings(source): - """Strip string lookup table (list) and replace values in source.""" - match = re.search(r'var *(_\w+)\=\["(.*?)"\];', source, re.DOTALL) - - if match: - varname, strings = match.groups() - startpoint = len(match.group(0)) - lookup = strings.split('","') - variable = '%s[%%d]' % varname - for index, value in enumerate(lookup): - source = source.replace(variable % index, '"%s"' % value) - return source[startpoint:] - return source - - -class Unbaser(object): - """Functor for a given base. Will efficiently convert - strings to natural numbers.""" - ALPHABET = { - 62 : '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ', - 95 : (' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ' - '[\]^_`abcdefghijklmnopqrstuvwxyz{|}~') - } - - def __init__(self, base): - self.base = base - - # If base can be handled by int() builtin, let it do it for us - if 2 <= base <= 36: - self.unbase = lambda string: int(string, base) - else: - # Build conversion dictionary cache - try: - self.dictionary = dict((cipher, index) for - index, cipher in enumerate(self.ALPHABET[base])) - except KeyError: - raise TypeError('Unsupported base encoding.') - - self.unbase = self._dictunbaser - - def __call__(self, string): - return self.unbase(string) - - def _dictunbaser(self, string): - """Decodes a value to an integer.""" - ret = 0 - for index, cipher in enumerate(string[::-1]): - ret += (self.base ** index) * self.dictionary[cipher] - return ret diff --git a/mitmproxy/contrib/jsbeautifier/unpackers/urlencode.py b/mitmproxy/contrib/jsbeautifier/unpackers/urlencode.py deleted file mode 100644 index 72d2bd1c..00000000 --- a/mitmproxy/contrib/jsbeautifier/unpackers/urlencode.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Trivial bookmarklet/escaped script detector for the javascript beautifier -# written by Einar Lielmanis <einar@jsbeautifier.org> -# rewritten in Python by Stefano Sanfilippo <a.little.coder@gmail.com> -# -# Will always return valid javascript: if `detect()` is false, `code` is -# returned, unmodified. -# -# usage: -# -# some_string = urlencode.unpack(some_string) -# - -"""Bookmarklet/escaped script unpacker.""" - -# Python 2 retrocompatibility -# pylint: disable=F0401 -# pylint: disable=E0611 -try: - from urllib import unquote_plus -except ImportError: - from urllib.parse import unquote_plus - -PRIORITY = 0 - -def detect(code): - """Detects if a scriptlet is urlencoded.""" - # the fact that script doesn't contain any space, but has %20 instead - # should be sufficient check for now. - return ' ' not in code and ('%20' in code or code.count('%') > 3) - -def unpack(code): - """URL decode `code` source string.""" - return unquote_plus(code) if detect(code) else code diff --git a/mitmproxy/main.py b/mitmproxy/main.py index 6d44108e..464c3897 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/main.py @@ -92,6 +92,7 @@ def mitmdump(args=None): # pragma: no cover if args.quiet: args.flow_detail = 0 + master = None try: dump_options = dump.Options(**cmdline.get_common_options(args)) dump_options.flow_detail = args.flow_detail @@ -110,7 +111,7 @@ def mitmdump(args=None): # pragma: no cover sys.exit(1) except (KeyboardInterrupt, _thread.error): pass - if master.has_errored: + if master is None or master.has_errored: print("mitmdump: errors occurred during run", file=sys.stderr) sys.exit(1) diff --git a/mitmproxy/onboarding/app.py b/mitmproxy/onboarding/app.py index e26efae8..491ddbfe 100644 --- a/mitmproxy/onboarding/app.py +++ b/mitmproxy/onboarding/app.py @@ -48,6 +48,7 @@ class PEM(tornado.web.RequestHandler): def get(self): p = os.path.join(self.request.master.options.cadir, self.filename) + p = os.path.expanduser(p) self.set_header("Content-Type", "application/x-x509-ca-cert") self.set_header( "Content-Disposition", diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py index e1ff7c47..2e350131 100644 --- a/mitmproxy/platform/__init__.py +++ b/mitmproxy/platform/__init__.py @@ -1,8 +1,9 @@ import sys +import re resolver = None -if sys.platform == "linux2": +if re.match(r"linux(?:2)?", sys.platform): from . import linux resolver = linux.Resolver elif sys.platform == "darwin": diff --git a/netlib/encoding.py b/netlib/encoding.py index da282194..9c8acff7 100644 --- a/netlib/encoding.py +++ b/netlib/encoding.py @@ -8,6 +8,7 @@ import collections from io import BytesIO import gzip import zlib +import brotli from typing import Union # noqa @@ -45,7 +46,7 @@ def decode(encoded, encoding, errors='strict'): decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate"): + if encoding in ("gzip", "deflate", "br"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except Exception as e: @@ -81,7 +82,7 @@ def encode(decoded, encoding, errors='strict'): encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate"): + if encoding in ("gzip", "deflate", "br"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except Exception as e: @@ -113,6 +114,14 @@ def encode_gzip(content): return s.getvalue() +def decode_brotli(content): + return brotli.decompress(content) + + +def encode_brotli(content): + return brotli.compress(content) + + def decode_deflate(content): """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -139,11 +148,13 @@ custom_decode = { "identity": identity, "gzip": decode_gzip, "deflate": decode_deflate, + "br": decode_brotli, } custom_encode = { "identity": identity, "gzip": encode_gzip, "deflate": encode_deflate, + "br": encode_brotli, } __all__ = ["encode", "decode"] diff --git a/netlib/http/message.py b/netlib/http/message.py index be35b8d1..ce92bab1 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -248,7 +248,7 @@ class Message(basetypes.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate" or "identity". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/netlib/http/request.py b/netlib/http/request.py index 061217a3..d59fead4 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -337,7 +337,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate"} + for e in {"gzip", "identity", "deflate", "br"} if e in accept_encoding ) ) diff --git a/netlib/strutils.py b/netlib/strutils.py index 8f27ebb7..4a46b6b1 100644 --- a/netlib/strutils.py +++ b/netlib/strutils.py @@ -69,7 +69,7 @@ def escape_control_characters(text, keep_spacing=True): return text.translate(trans) -def bytes_to_escaped_str(data, keep_spacing=False): +def bytes_to_escaped_str(data, keep_spacing=False, escape_single_quotes=False): """ Take bytes and return a safe string that can be displayed to the user. @@ -86,6 +86,8 @@ def bytes_to_escaped_str(data, keep_spacing=False): # We always insert a double-quote here so that we get a single-quoted string back # https://stackoverflow.com/questions/29019340/why-does-python-use-different-quotes-for-representing-strings-depending-on-their ret = repr(b'"' + data).lstrip("b")[2:-1] + if not escape_single_quotes: + ret = re.sub(r"(?<!\\)(\\\\)*\\'", lambda m: (m.group(1) or "") + "'", ret) if keep_spacing: ret = re.sub( r"(?<!\\)(\\\\)*\\([nrt])", diff --git a/pathod/language/base.py b/pathod/language/base.py index 25f3fd1a..39155858 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -136,7 +136,7 @@ class TokValueLiteral(_TokValueLiteral): def spec(self): inner = strutils.bytes_to_escaped_str(self.val) - inner = inner.replace(r"\'", r"\x27") + inner = inner.replace(r"'", r"\x27") return "'" + inner + "'" @@ -148,7 +148,7 @@ class TokValueNakedLiteral(_TokValueLiteral): return e.setParseAction(lambda x: cls(*x)) def spec(self): - return strutils.bytes_to_escaped_str(self.val) + return strutils.bytes_to_escaped_str(self.val, escape_single_quotes=True) class TokValueGenerate(Token): @@ -166,7 +166,7 @@ class TokValueGenerate(Token): def freeze(self, settings): g = self.get_generator(settings) - return TokValueLiteral(strutils.bytes_to_escaped_str(g[:])) + return TokValueLiteral(strutils.bytes_to_escaped_str(g[:], escape_single_quotes=True)) @classmethod def expr(cls): @@ -578,4 +578,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode()))) + return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True))) diff --git a/pathod/pathoc.py b/pathod/pathoc.py index c6783878..5831ba3e 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -444,7 +444,7 @@ class Pathoc(tcp.TCPClient): finally: if resp: lg("<< %s %s: %s bytes" % ( - resp.status_code, strutils.bytes_to_escaped_str(resp.reason.encode()), len(resp.content) + resp.status_code, strutils.escape_control_characters(resp.reason) if resp.reason else "", len(resp.content) )) if resp.status_code in self.ignorecodes: lg.suppress() diff --git a/release/rtool.py b/release/rtool.py index 4e43eaef..45e1416e 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,29 +1,30 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import absolute_import, print_function, division -from os.path import join + import contextlib +import fnmatch import os +import platform +import runpy +import shlex import shutil import subprocess -import re -import shlex -import runpy -import zipfile +import sys import tarfile -import platform +import zipfile +from os.path import join, abspath, normpath, dirname, exists, basename + import click import pysftp -import fnmatch # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -import sys - if platform.system() == "Windows": VENV_BIN = "Scripts" else: VENV_BIN = "bin" +# ZipFile and tarfile have slightly different APIs. Fix that. if platform.system() == "Windows": def Archive(name): a = zipfile.ZipFile(name, "w") @@ -33,13 +34,13 @@ else: def Archive(name): return tarfile.open(name, "w:gz") -RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) -DIST_DIR = join(RELEASE_DIR, "dist") -ROOT_DIR = os.path.normpath(join(RELEASE_DIR, "..")) -RELEASE_SPEC_DIR = join(RELEASE_DIR, "specs") -VERSION_FILE = join(ROOT_DIR, "netlib/version.py") +ROOT_DIR = abspath(join(dirname(__file__), "..")) +RELEASE_DIR = join(ROOT_DIR, "release") BUILD_DIR = join(RELEASE_DIR, "build") +DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") PYINSTALLER_DIST = join(BUILD_DIR, "binaries") @@ -47,27 +48,35 @@ VENV_DIR = join(BUILD_DIR, "venv") VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") -project = { - "name": "mitmproxy", - "tools": ["pathod", "pathoc", "mitmproxy", "mitmdump", "mitmweb"], - "bdists": { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] - }, - "dir": ROOT_DIR, - "python_version": "py2.py3", +# Project Configuration +VERSION_FILE = join(ROOT_DIR, "netlib", "version.py") +PROJECT_NAME = "mitmproxy" +PYTHON_VERSION = "py2.py3" +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] } if platform.system() == "Windows": - project["tools"].remove("mitmproxy") - project["bdists"]["mitmproxy"].remove("mitmproxy") + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in BDISTS.values() + for tool in tools +] -def get_version(): +def get_version() -> str: return runpy.run_path(VERSION_FILE)["VERSION"] -def get_snapshot_version(): - last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit(b"-", 2) +def git(args: str) -> str: + with chdir(ROOT_DIR): + return subprocess.check_output(["git"] + shlex.split(args)).decode() + + +def get_snapshot_version() -> str: + last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) tag_dist = int(tag_dist) if tag_dist == 0: return get_version() @@ -76,11 +85,11 @@ def get_snapshot_version(): return "{version}dev{tag_dist:04}-0x{commit}".format( version=get_version(), # this should already be the next version tag_dist=tag_dist, - commit=commit.decode() + commit=commit ) -def archive_name(project): +def archive_name(bdist: str) -> str: platform_tag = { "Darwin": "osx", "Windows": "win32", @@ -91,18 +100,18 @@ def archive_name(project): else: ext = "tar.gz" return "{project}-{version}-{platform}.{ext}".format( - project=project, + project=bdist, version=get_version(), platform=platform_tag, ext=ext ) -def wheel_name(): +def wheel_name() -> str: return "{project}-{version}-{py_version}-none-any.whl".format( - project=project["name"], + project=PROJECT_NAME, version=get_version(), - py_version=project["python_version"] + py_version=PYTHON_VERSION ) @@ -119,18 +128,13 @@ def empty_pythonpath(): @contextlib.contextmanager -def chdir(path): +def chdir(path: str): old_dir = os.getcwd() os.chdir(path) yield os.chdir(old_dir) -def git(args): - with chdir(ROOT_DIR): - return subprocess.check_output(["git"] + shlex.split(args)) - - @click.group(chain=True) def cli(): """ @@ -147,95 +151,79 @@ def contributors(): with chdir(ROOT_DIR): print("Updating CONTRIBUTORS...") contributors_data = git("shortlog -n -s") - with open("CONTRIBUTORS", "w") as f: - f.write(contributors_data) + with open("CONTRIBUTORS", "wb") as f: + f.write(contributors_data.encode()) -@cli.command("set-version") -@click.argument('version') -def set_version(version): +@cli.command("wheel") +def make_wheel(): """ - Update version information - """ - print("Update versions...") - version = ", ".join(version.split(".")) - print("Update %s..." % VERSION_FILE) - with open(VERSION_FILE, "rb") as f: - content = f.read() - new_content = re.sub( - r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, - content - ) - with open(VERSION_FILE, "wb") as f: - f.write(new_content) - - -@cli.command("wheels") -def wheels(): - """ - Build wheels + Build wheel """ with empty_pythonpath(): - print("Building release...") - if os.path.exists(DIST_DIR): + if exists(DIST_DIR): shutil.rmtree(DIST_DIR) - print("Creating wheel for %s ..." % project["name"]) + print("Creating wheel...") subprocess.check_call( [ "python", "./setup.py", "-q", "bdist_wheel", "--dist-dir", DIST_DIR, "--universal" ], - cwd=project["dir"] + cwd=ROOT_DIR ) print("Creating virtualenv for test install...") - if os.path.exists(VENV_DIR): + if exists(VENV_DIR): shutil.rmtree(VENV_DIR) subprocess.check_call(["virtualenv", "-q", VENV_DIR]) with chdir(DIST_DIR): - print("Installing %s..." % project["name"]) + print("Install wheel into virtualenv...") # lxml... if platform.system() == "Windows" and sys.version_info[0] == 3: - subprocess.check_call([VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]) + subprocess.check_call( + [VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"] + ) subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()]) - print("Running binaries...") - for tool in project["tools"]: + print("Running tools...") + for tool in TOOLS: tool = join(VENV_DIR, VENV_BIN, tool) print("> %s --version" % tool) - print(subprocess.check_output([tool, "--version"])) + print(subprocess.check_output([tool, "--version"]).decode()) print("Virtualenv available for further testing:") - print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate"))) + print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate"))) @cli.command("bdist") -@click.option("--use-existing-wheels/--no-use-existing-wheels", default=False) +@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False) @click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.1.1") +@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1") @click.pass_context -def bdist(ctx, use_existing_wheels, pyinstaller_version): +def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version): """ Build a binary distribution """ - if os.path.exists(PYINSTALLER_TEMP): + if exists(PYINSTALLER_TEMP): shutil.rmtree(PYINSTALLER_TEMP) - if os.path.exists(PYINSTALLER_DIST): + if exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) - if not use_existing_wheels: - ctx.invoke(wheels) + if not use_existing_wheel: + ctx.invoke(make_wheel) - print("Installing PyInstaller...") - subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version]) + print("Installing PyInstaller and setuptools...") + subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version]) + print(subprocess.check_output([VENV_PIP, "freeze"]).decode()) - for bdist_project, tools in project["bdists"].items(): - with Archive(join(DIST_DIR, archive_name(bdist_project))) as archive: + for bdist, tools in BDISTS.items(): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: for tool in tools: # This is PyInstaller, so it messes up paths. # We need to make sure that we are in the spec folder. - with chdir(RELEASE_SPEC_DIR): + with chdir(PYINSTALLER_SPEC): print("Building %s binary..." % tool) subprocess.check_call( [ @@ -255,10 +243,10 @@ def bdist(ctx, use_existing_wheels, pyinstaller_version): if platform.system() == "Windows": executable += ".exe" print("> %s --version" % executable) - subprocess.check_call([executable, "--version"]) + print(subprocess.check_output([executable, "--version"]).decode()) - archive.add(executable, os.path.basename(executable)) - print("Packed {}.".format(archive_name(bdist_project))) + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) @cli.command("upload-release") @@ -298,89 +286,49 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, username=user, private_key=private_key, private_key_pass=private_key_password) as sftp: - - dir_name = "snapshots/v{}".format(get_version()) - sftp.makedirs(dir_name) - with sftp.cd(dir_name): - files = [] - if wheel: - files.append(wheel_name()) - for bdist in project["bdists"].keys(): + dir_name = "snapshots/v{}".format(get_version()) + sftp.makedirs(dir_name) + with sftp.cd(dir_name): + files = [] + if wheel: + files.append(wheel_name()) + if bdist: + for bdist in BDISTS.keys(): files.append(archive_name(bdist)) - for f in files: - local_path = join(DIST_DIR, f) - remote_filename = f.replace(get_version(), get_snapshot_version()) - symlink_path = "../{}".format(f.replace(get_version(), "latest")) - - # Delete old versions - old_version = f.replace(get_version(), "*") - for f_old in sftp.listdir(): - if fnmatch.fnmatch(f_old, old_version): - print("Removing {}...".format(f_old)) - sftp.remove(f_old) - - # Upload new version - print("Uploading {} as {}...".format(f, remote_filename)) - with click.progressbar(length=os.stat(local_path).st_size) as bar: - sftp.put( - local_path, - "." + remote_filename, - callback=lambda done, total: bar.update(done - bar.pos) - ) - # We hide the file during upload. - sftp.rename("." + remote_filename, remote_filename) - - # update symlink for the latest release - if sftp.lexists(symlink_path): - print("Removing {}...".format(symlink_path)) - sftp.remove(symlink_path) - sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) - - -@cli.command("wizard") -@click.option('--next-version', prompt=True) -@click.option('--username', prompt="PyPI Username") -@click.password_option(confirmation_prompt=False, prompt="PyPI Password") -@click.option('--repository', default="pypi") -@click.pass_context -def wizard(ctx, next_version, username, password, repository): - """ - Interactive Release Wizard - """ - is_dirty = git("status --porcelain") - if is_dirty: - raise RuntimeError("Repository is not clean.") - - # update contributors file - ctx.invoke(contributors) - - # Build test release - ctx.invoke(bdist) - - try: - click.confirm("Please test the release now. Is it ok?", abort=True) - except click.Abort: - # undo changes - git("checkout CONTRIBUTORS") - raise - - # Everything ok - let's ship it! - git("tag v{}".format(get_version())) - git("push --tags") - ctx.invoke( - upload_release, - username=username, password=password, repository=repository - ) - - click.confirm("Now please wait until CI has built binaries. Finished?") - - # version bump commit - ctx.invoke(set_version, version=next_version) - git("commit -a -m \"bump version\"") - git("push") + for f in files: + local_path = join(DIST_DIR, f) + remote_filename = f.replace(get_version(), get_snapshot_version()) + symlink_path = "../{}".format(f.replace(get_version(), "latest")) + + # Upload new version + print("Uploading {} as {}...".format(f, remote_filename)) + with click.progressbar(length=os.stat(local_path).st_size) as bar: + # We hide the file during upload + sftp.put( + local_path, + "." + remote_filename, + callback=lambda done, total: bar.update(done - bar.pos) + ) - click.echo("All done!") + # Delete old versions + old_version = f.replace(get_version(), "*") + for f_old in sftp.listdir(): + if fnmatch.fnmatch(f_old, old_version): + print("Removing {}...".format(f_old)) + sftp.remove(f_old) + + # Show new version + sftp.rename("." + remote_filename, remote_filename) + + # update symlink for the latest release + if sftp.lexists(symlink_path): + print("Removing {}...".format(symlink_path)) + sftp.remove(symlink_path) + if f != wheel_name(): + # "latest" isn't a proper wheel version, so this could not be installed. + # https://github.com/mitmproxy/mitmproxy/issues/1065 + sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) if __name__ == "__main__": @@ -67,10 +67,12 @@ setup( "configargparse>=0.10, <0.11", "construct>=2.5.2, <2.6", "cryptography>=1.3, <1.5", + "cssutils>=1.0.1, <1.1", "Flask>=0.10.1, <0.12", "h2>=2.4.0, <3", "html2text>=2016.1.8, <=2016.5.29", "hyperframe>=4.0.1, <5", + "jsbeautifier>=1.6.3, <1.7", "lxml>=3.5.0, <=3.6.0", # no wheels for 3.6.1 yet. "Pillow>=3.2, <3.4", "passlib>=1.6.5, <1.7", @@ -83,6 +85,7 @@ setup( "tornado>=4.3, <4.5", "urwid>=1.3.1, <1.4", "watchdog>=0.8.3, <0.9", + "brotlipy>=0.3.0, <0.4", ], extras_require={ ':sys_platform == "win32"': [ @@ -110,7 +113,6 @@ setup( "sphinx_rtd_theme>=0.1.9, <0.2", ], 'contentviews': [ - "cssutils>=1.0.1, <1.1", # TODO: Find Python 3 replacements # "protobuf>=2.6.1, <2.7", # "pyamf>=0.8.0, <0.9", diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index aad53b37..66cad47b 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -78,6 +78,7 @@ class TestContentView: v = cv.ViewHTMLOutline() s = b"<html><br><br></br><p>one</p></html>" assert v(s) + assert v(b'\xfe') def test_view_json(self): cv.VIEW_CUTOFF = 100 @@ -106,9 +107,10 @@ class TestContentView: def test_view_javascript(self): v = cv.ViewJavaScript() - assert v("[1, 2, 3]") - assert v("[1, 2, 3") - assert v("function(a){[1, 2, 3]}") + assert v(b"[1, 2, 3]") + assert v(b"[1, 2, 3") + assert v(b"function(a){[1, 2, 3]}") + assert v(b"\xfe") # invalid utf-8 def test_view_css(self): v = cv.ViewCSS() diff --git a/test/netlib/test_encoding.py b/test/netlib/test_encoding.py index a5e81379..08e69ec5 100644 --- a/test/netlib/test_encoding.py +++ b/test/netlib/test_encoding.py @@ -21,6 +21,18 @@ def test_gzip(): encoding.decode(b"bogus", "gzip") +def test_brotli(): + assert b"string" == encoding.decode( + encoding.encode( + b"string", + "br" + ), + "br" + ) + with tutils.raises(ValueError): + encoding.decode(b"bogus", "br") + + def test_deflate(): assert b"string" == encoding.decode( encoding.encode( diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index 7c3eacc6..52299e59 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -48,9 +48,12 @@ def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" - assert strutils.bytes_to_escaped_str(b"'") == r"\'" + assert strutils.bytes_to_escaped_str(b"'") == r"'" assert strutils.bytes_to_escaped_str(b'"') == r'"' + assert strutils.bytes_to_escaped_str(b"'", escape_single_quotes=True) == r"\'" + assert strutils.bytes_to_escaped_str(b'"', escape_single_quotes=True) == r'"' + assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" |