diff options
| -rw-r--r-- | mitmproxy/contentviews/css.py | 8 | ||||
| -rw-r--r-- | mitmproxy/contentviews/javascript.py | 49 | ||||
| -rw-r--r-- | mitmproxy/contrib/README | 7 | ||||
| -rw-r--r-- | mitmproxy/utils/strutils.py | 22 | ||||
| -rw-r--r-- | release/hooks/hook-passlib.py | 1 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | setup.py | 1 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_javascript.py | 24 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_js_data/simple-formatted.js | 68 | ||||
| -rw-r--r-- | test/mitmproxy/contentviews/test_js_data/simple.js | 8 | ||||
| -rw-r--r-- | test/mitmproxy/utils/test_strutils.py | 8 |
11 files changed, 164 insertions, 34 deletions
diff --git a/mitmproxy/contentviews/css.py b/mitmproxy/contentviews/css.py index 8fa09ed3..30a583da 100644 --- a/mitmproxy/contentviews/css.py +++ b/mitmproxy/contentviews/css.py @@ -14,10 +14,10 @@ A custom CSS prettifier. Compared to other prettifiers, its main features are: """ CSS_SPECIAL_AREAS = ( - ("'", strutils.NO_ESCAPE + "'"), - ('"', strutils.NO_ESCAPE + '"'), - (r"/\*", r"\*/"), - ("//", "$") + "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", + '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"', + r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + "//" + strutils.SINGLELINE_CONTENT + "$" ) CSS_SPECIAL_CHARS = "{};:" diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index c2fab875..1d671fe6 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -1,6 +1,47 @@ -import jsbeautifier +import io +import re -from . import base +from mitmproxy.utils import strutils +from mitmproxy.contentviews import base + +DELIMITERS = '{};\n' +SPECIAL_AREAS = ( + r"(?<=[^\w\s)])\s*/(?:[^\n/]|(?<!\\)(?:\\\\)*\\/)+?/(?=[gimsuy]{0,6}\s*(?:[;,).\n]|$))", + r"'" + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + "'", + r'"' + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + '"', + r'`' + strutils.MULTILINE_CONTENT + strutils.NO_ESCAPE + '`', + r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"//" + strutils.SINGLELINE_CONTENT + "$", + r"for\(" + strutils.SINGLELINE_CONTENT + "\)", +) + + +def beautify(data): + data = strutils.escape_special_areas( + data, + SPECIAL_AREAS, + DELIMITERS + ) + + data = re.sub(r"\s*{\s*(?!};)", " {\n", data) + data = re.sub(r"\s*;\s*", ";\n", data) + data = re.sub(r"(?<!{)\s*}(;)?\s*", r"\n}\1\n", data) + + beautified = io.StringIO() + indent_level = 0 + + for line in data.splitlines(True): + if line.endswith("{\n"): + beautified.write(" " * 2 * indent_level + line) + indent_level += 1 + elif line.startswith("}"): + indent_level -= 1 + beautified.write(" " * 2 * indent_level + line) + else: + beautified.write(" " * 2 * indent_level + line) + + data = strutils.unescape_special_areas(beautified.getvalue()) + return data class ViewJavaScript(base.View): @@ -13,8 +54,6 @@ class ViewJavaScript(base.View): ] def __call__(self, data, **metadata): - opts = jsbeautifier.default_options() - opts.indent_size = 2 data = data.decode("utf-8", "replace") - res = jsbeautifier.beautify(data, opts) + res = beautify(data) return "JavaScript", base.format_text(res) diff --git a/mitmproxy/contrib/README b/mitmproxy/contrib/README index e5ce11da..fc44eed3 100644 --- a/mitmproxy/contrib/README +++ b/mitmproxy/contrib/README @@ -1,14 +1,9 @@ Contribs: -jsbeautifier, git checkout 25/03/12, MIT license - - Removed test directories - - Disabled packers through a single-line modification (see "# CORTESI" - comment) - wbxml - https://github.com/davidpshaw/PyWBXMLDecoder tls, BSD license - https://github.com/mhils/tls/tree/mitmproxy - - limited to required files.
\ No newline at end of file + - limited to required files. diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 37bed7de..71d1c54c 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,7 +1,7 @@ import io import re import codecs -from typing import AnyStr, Optional, cast, Iterable, Tuple +from typing import AnyStr, Optional, cast, Iterable def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: @@ -153,11 +153,14 @@ def _restore_from_private_code_plane(matchobj): NO_ESCAPE = r"(?<!\\)(?:\\\\)*" +MULTILINE_CONTENT = r"[\s\S]*?" +SINGLELINE_CONTENT = r".*?" +MULTILINE_CONTENT_LINE_CONTINUATION = r"(?:.|(?<=\\)\n)*?" def split_special_areas( data: str, - area_delimiter: Iterable[Tuple[str, str]], + area_delimiter: Iterable[str], ): """ Split a string of code into a [code, special area, code, special area, ..., code] list. @@ -166,18 +169,13 @@ def split_special_areas( >>> split_special_areas( >>> "test /* don't modify me */ foo", - >>> [(r"/\*", r"\*/")]) # (left delimiter regex, right delimiter regex) + >>> [r"/\*[\s\S]*?\*/"]) # (regex matching comments) ["test ", "/* don't modify me */", " foo"] "".join(split_special_areas(x, ...)) == x always holds true. """ - patterns = "|".join( - r"{lchar}[\s\S]*?{rchar}".format( - lchar=a, - rchar=b, - ) for (a, b) in area_delimiter) return re.split( - "({})".format(patterns), + "({})".format("|".join(area_delimiter)), data, flags=re.MULTILINE ) @@ -185,7 +183,7 @@ def split_special_areas( def escape_special_areas( data: str, - area_delimiter: Iterable[Tuple[str, str]], + area_delimiter: Iterable[str], control_characters, ): """ @@ -200,11 +198,11 @@ def escape_special_areas( >>> print(x) if (true) { console.log('{}'); } - >>> x = escape_special_areas(x, "{", [("'", "'")]) + >>> x = escape_special_areas(x, "{", ["'" + SINGLELINE_CONTENT + "'"]) >>> print(x) if (true) { console.log('�}'); } >>> x = re.sub(r"\s*{\s*", " {\n ", x) - >>> x = unescape_special_areas(x, "{", [("'", "'")]) + >>> x = unescape_special_areas(x) >>> print(x) if (true) { console.log('{}'); } diff --git a/release/hooks/hook-passlib.py b/release/hooks/hook-passlib.py new file mode 100644 index 00000000..4f1efbb8 --- /dev/null +++ b/release/hooks/hook-passlib.py @@ -0,0 +1 @@ +hiddenimports = ["configparser"] @@ -1,7 +1,7 @@ [flake8] max-line-length = 140 max-complexity = 25 -ignore = E251,C901,W503 +ignore = E251,C901,W503,W292 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/* addons = file,open,basestring,xrange,unicode,long,cmp @@ -68,7 +68,6 @@ setup( "h2>=3.0, <4", "html2text>=2016.1.8, <=2016.9.19", "hyperframe>=5.0, <6", - "jsbeautifier>=1.6.3, <1.7", "kaitaistruct>=0.7, <0.8", "ldap3>=2.2.0, <2.4", "passlib>=1.6.5, <1.8", diff --git a/test/mitmproxy/contentviews/test_javascript.py b/test/mitmproxy/contentviews/test_javascript.py index 43039c93..23dd106e 100644 --- a/test/mitmproxy/contentviews/test_javascript.py +++ b/test/mitmproxy/contentviews/test_javascript.py @@ -1,10 +1,32 @@ +import pytest + from mitmproxy.contentviews import javascript +from mitmproxy.test import tutils from . import full_eval +data = tutils.test_data.push("mitmproxy/contentviews/test_js_data/") + def test_view_javascript(): v = full_eval(javascript.ViewJavaScript()) assert v(b"[1, 2, 3]") assert v(b"[1, 2, 3") - assert v(b"function(a){[1, 2, 3]}") + assert v(b"function(a){[1, 2, 3]}") == ("JavaScript", [ + [('text', 'function(a) {')], + [('text', ' [1, 2, 3]')], + [('text', '}')] + ]) assert v(b"\xfe") # invalid utf-8 + + +@pytest.mark.parametrize("filename", [ + "simple.js", +]) +def test_format_xml(filename): + path = data.path(filename) + with open(path) as f: + input = f.read() + with open("-formatted.".join(path.rsplit(".", 1))) as f: + expected = f.read() + js = javascript.beautify(input) + assert js == expected diff --git a/test/mitmproxy/contentviews/test_js_data/simple-formatted.js b/test/mitmproxy/contentviews/test_js_data/simple-formatted.js new file mode 100644 index 00000000..2b665f02 --- /dev/null +++ b/test/mitmproxy/contentviews/test_js_data/simple-formatted.js @@ -0,0 +1,68 @@ +/* _GlobalPrefix_ */ +this.gbar_=this.gbar_||{}; +(function(_) { + var window=this; + +/* _Module_:sy25 */ + try { + var Mn=function(){}; + _.y(Mn,Error); + _.Nn=function() { + this.b="pending"; + this.B=[]; + this.w=this.C=void 0 + }; + _.fe(_.Nn); + var On=function() { + _.qa.call(this,"Multiple attempts to set the state of this Result") + }; + _.y(On,_.qa); + _.Nn.prototype.ta=function() { + return this.C + }; + _.Pn=function(a,c,d) { + "pending"==a.b?a.B.push( { + hb:c,scope:d||null + } + ):c.call(d,a) + }; + _.Nn.prototype.A=function(a) { + if("pending"==this.b)this.C=a,this.b="success",Qn(this); + else if(!Rn(this))throw new On; + }; + _.Nn.prototype.o=function(a) { + if("pending"==this.b)this.w=a,this.b="error",Qn(this); + else if(!Rn(this))throw new On; + }; + var Qn=function(a) { + var c=a.B; + a.B=[]; + for(var d=0;d<c.length;d++) { + var e=c[d]; + e.hb.call(e.scope,a) + } + + }; + _.Nn.prototype.cancel=function() { + return"pending"==this.b?(this.o(new Mn),!0):!1 + }; + var Rn=function(a) { + return"error"==a.b&&a.w instanceof Mn + }; + _.Nn.prototype.then=function(a,c,d) { + var e,f,g=new _.ie(function(a,c) { + e=a; + f=c + } + ); + _.Pn(this,function(a) { + Rn(a)?g.cancel():"success"==a.b?e(a.ta()):"error"==a.b&&f(a.w) + } + ); + return g.then(a,c,d) + }; + + } + catch(e) { + _._DumpException(e) + } diff --git a/test/mitmproxy/contentviews/test_js_data/simple.js b/test/mitmproxy/contentviews/test_js_data/simple.js new file mode 100644 index 00000000..66a56d8b --- /dev/null +++ b/test/mitmproxy/contentviews/test_js_data/simple.js @@ -0,0 +1,8 @@ +/* _GlobalPrefix_ */ +this.gbar_=this.gbar_||{};(function(_){var window=this; +/* _Module_:sy25 */ +try{ +var Mn=function(){};_.y(Mn,Error);_.Nn=function(){this.b="pending";this.B=[];this.w=this.C=void 0};_.fe(_.Nn);var On=function(){_.qa.call(this,"Multiple attempts to set the state of this Result")};_.y(On,_.qa);_.Nn.prototype.ta=function(){return this.C};_.Pn=function(a,c,d){"pending"==a.b?a.B.push({hb:c,scope:d||null}):c.call(d,a)};_.Nn.prototype.A=function(a){if("pending"==this.b)this.C=a,this.b="success",Qn(this);else if(!Rn(this))throw new On;}; +_.Nn.prototype.o=function(a){if("pending"==this.b)this.w=a,this.b="error",Qn(this);else if(!Rn(this))throw new On;};var Qn=function(a){var c=a.B;a.B=[];for(var d=0;d<c.length;d++){var e=c[d];e.hb.call(e.scope,a)}};_.Nn.prototype.cancel=function(){return"pending"==this.b?(this.o(new Mn),!0):!1};var Rn=function(a){return"error"==a.b&&a.w instanceof Mn}; _.Nn.prototype.then=function(a,c,d){var e,f,g=new _.ie(function(a,c){e=a;f=c});_.Pn(this,function(a){Rn(a)?g.cancel():"success"==a.b?e(a.ta()):"error"==a.b&&f(a.w)});return g.then(a,c,d)}; + +}catch(e){_._DumpException(e)} diff --git a/test/mitmproxy/utils/test_strutils.py b/test/mitmproxy/utils/test_strutils.py index 7ec72e4e..dfe2c620 100644 --- a/test/mitmproxy/utils/test_strutils.py +++ b/test/mitmproxy/utils/test_strutils.py @@ -99,8 +99,8 @@ def test_hexdump(): ESCAPE_QUOTES = [ - ("'", strutils.NO_ESCAPE + "'"), - ('"', strutils.NO_ESCAPE + '"') + "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", + '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"' ] @@ -113,11 +113,11 @@ def test_split_special_areas(): ) == ["foo ", "'b\\'a\"r'", " baz"] assert strutils.split_special_areas( "foo\n/*bar\nbaz*/\nqux", - [(r'/\*', r'\*/')] + [r'/\*[\s\S]+?\*/'] ) == ["foo\n", "/*bar\nbaz*/", "\nqux"] assert strutils.split_special_areas( "foo\n//bar\nbaz", - [(r'//', r'$')] + [r'//.+$'] ) == ["foo\n", "//bar", "\nbaz"] |