Removed the custom lexer in favor of using pyparsing.

4 files changed, 134 insertions, 111 deletions

diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py
index ad475fba..ae4c400c 100644
--- a/test/mitmproxy/test_command.py
+++ b/test/mitmproxy/test_command.py
@@ -115,12 +115,9 @@ class TestCommand:
             [
                 "foo bar",
                 [
-                    command.ParseResult(
-                        value = "foo", type = mitmproxy.types.Cmd, valid = False
-                    ),
-                    command.ParseResult(
-                        value = "bar", type = mitmproxy.types.Unknown, valid = False
-                    )
+                    command.ParseResult(value = "foo", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "bar", type = mitmproxy.types.Unknown, valid = False)
                 ],
                 [],
             ],
@@ -128,6 +125,7 @@ class TestCommand:
                 "cmd1 'bar",
                 [
                     command.ParseResult(value = "cmd1", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "'bar", type = str, valid = True)
                 ],
                 [],
@@ -146,6 +144,7 @@ class TestCommand:
                 "cmd3 1",
                 [
                     command.ParseResult(value = "cmd3", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "1", type = int, valid = True),
                 ],
                 []
@@ -154,28 +153,27 @@ class TestCommand:
                 "cmd3 ",
                 [
                     command.ParseResult(value = "cmd3", type = mitmproxy.types.Cmd, valid = True),
-                    command.ParseResult(value = "", type = int, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                 ],
-                []
+                ['int']
             ],
             [
                 "subcommand ",
                 [
-                    command.ParseResult(
-                        value = "subcommand", type = mitmproxy.types.Cmd, valid = True,
-                    ),
-                    command.ParseResult(value = "", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = "subcommand", type = mitmproxy.types.Cmd, valid = True,),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                 ],
-                ["arg"],
+                ["cmd", "arg"],
             ],
             [
                 "subcommand cmd3 ",
                 [
                     command.ParseResult(value = "subcommand", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "cmd3", type = mitmproxy.types.Cmd, valid = True),
-                    command.ParseResult(value = "", type = int, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                 ],
-                []
+                ["int"]
             ],
             [
                 "cmd4",
@@ -188,22 +186,15 @@ class TestCommand:
                 "cmd4 ",
                 [
                     command.ParseResult(value = "cmd4", type = mitmproxy.types.Cmd, valid = True),
-                    command.ParseResult(value = "", type = int, valid = False),
-                ],
-                ["str", "path"]
-            ],
-            [
-                "cmd4 1",
-                [
-                    command.ParseResult(value = "cmd4", type = mitmproxy.types.Cmd, valid = True),
-                    command.ParseResult(value = "1", type = int, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                 ],
-                ["str", "path"]
+                ["int", "str", "path"]
             ],
             [
                 "cmd4 1",
                 [
                     command.ParseResult(value = "cmd4", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "1", type = int, valid = True),
                 ],
                 ["str", "path"]
@@ -219,14 +210,15 @@ class TestCommand:
                 "flow ",
                 [
                     command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True),
-                    command.ParseResult(value = "", type = flow.Flow, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                 ],
-                ["str"]
+                ["flow", "str"]
             ],
             [
                 "flow x",
                 [
                     command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "x", type = flow.Flow, valid = False),
                 ],
                 ["str"]
@@ -235,15 +227,17 @@ class TestCommand:
                 "flow x ",
                 [
                     command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "x", type = flow.Flow, valid = False),
-                    command.ParseResult(value = "", type = str, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                 ],
-                []
+                ["str"]
             ],
             [
                 "flow \"one two",
                 [
                     command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = "\"one two", type = flow.Flow, valid = False),
                 ],
                 ["str"]
@@ -252,11 +246,112 @@ class TestCommand:
                 "flow \"three four\"",
                 [
                     command.ParseResult(value = "flow", type = mitmproxy.types.Cmd, valid = True),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
                     command.ParseResult(value = '"three four"', type = flow.Flow, valid = False),
                 ],
                 ["str"]
             ],
+            [
+                "spaces '    '",
+                [
+                    command.ParseResult(value = "spaces", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "'    '", type = mitmproxy.types.Unknown, valid = False)
+                ],
+                [],
+            ],
+            [
+                'spaces2 "    "',
+                [
+                    command.ParseResult(value = "spaces2", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = '"    "', type = mitmproxy.types.Unknown, valid = False)
+                ],
+                [],
+            ],
+            [
+                '"abc"',
+                [
+                    command.ParseResult(value = '"abc"', type = mitmproxy.types.Cmd, valid = False),
+                ],
+                [],
+            ],
+            [
+                "'def'",
+                [
+                    command.ParseResult(value = "'def'", type = mitmproxy.types.Cmd, valid = False),
+                ],
+                [],
+            ],
+            [
+                "cmd10 'a' \"b\" c",
+                [
+                    command.ParseResult(value = "cmd10", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "'a'", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = '"b"', type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "c", type = mitmproxy.types.Unknown, valid = False),
+                ],
+                [],
+            ],
+            [
+                "cmd11 'a \"b\" c'",
+                [
+                    command.ParseResult(value = "cmd11", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "'a \"b\" c'", type = mitmproxy.types.Unknown, valid = False),
+                ],
+                [],
+            ],
+            [
+                'cmd12 "a \'b\' c"',
+                [
+                    command.ParseResult(value = "cmd12", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = '"a \'b\' c"', type = mitmproxy.types.Unknown, valid = False),
+                ],
+                [],
+            ],
+            [
+                r'cmd13 "a \"b\" c"',
+                [
+                    command.ParseResult(value = "cmd13", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = r'"a \"b\" c"', type = mitmproxy.types.Unknown, valid = False),
+                ],
+                [],
+            ],
+            [
+                r"cmd14 'a \'b\' c'",
+                [
+                    command.ParseResult(value = "cmd14", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = r"'a \'b\' c'", type = mitmproxy.types.Unknown, valid = False),
+                ],
+                [],
+            ],
+            [
+                "    spaces_at_the_begining_are_stripped",
+                [
+                    command.ParseResult(value = "spaces_at_the_begining_are_stripped", type = mitmproxy.types.Cmd, valid = False),
+                ],
+                [],
+            ],
+            [
+                "    spaces_at_the_begining_are_stripped but_not_at_the_end      ",
+                [
+                    command.ParseResult(value = "spaces_at_the_begining_are_stripped", type = mitmproxy.types.Cmd, valid = False),
+                    command.ParseResult(value = " ", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "but_not_at_the_end", type = mitmproxy.types.Unknown, valid = False),
+                    command.ParseResult(value = "      ", type = mitmproxy.types.Unknown, valid = False),
+                ],
+                [],
+            ],
+
         ]
+
         with taddons.context() as tctx:
             tctx.master.addons.add(TAddon())
             for s, expected, expectedremain in tests:
diff --git a/test/mitmproxy/test_lexer.py b/test/mitmproxy/test_lexer.py
deleted file mode 100644
index 19ef155b..00000000
--- a/test/mitmproxy/test_lexer.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from mitmproxy import lexer
-import pytest
-import io
-
-
-class TestScripts:
-
-    def test_simple(self):
-
-        cases = [
-            {
-                "text": r'abc',
-                "result": ['abc']
-            },
-            {
-                "text": r'"Hello \" Double Quotes"',
-                "result": ['"Hello \\" Double Quotes"']
-            },
-            {
-                "text": r"'Hello \' Single Quotes'",
-                "result": ["'Hello \\' Single Quotes'"]
-            },
-            {
-                "text": r'"\""',
-                "result": ['"\\""']
-            },
-            {
-                "text": r'abc "def\" \x bla \z \\ \e \ " xpto',
-                "result": ['abc', '"def\\" \\x bla \\z \\\\ \\e \\ "', 'xpto']
-            },
-            {
-                "text": r'',
-                "result": []
-            },
-            {
-                "text": r' ',
-                "result": []
-            },
-            {
-                "text": r'  ',
-                "result": []
-            },
-            {
-                "text": r'Space in the end ',
-                "result": ['Space', 'in', 'the', 'end']
-            },
-            {
-                "text": '\n\n\rHello\n World With Spaces\n\n',
-                "result": ['Hello', 'World', 'With', 'Spaces']
-            },
-            {
-                "text": r'\" Escaping characters without reason',
-                "result": ['\\"', 'Escaping', 'characters', 'without', 'reason']
-            },
-        ]
-
-        for t in cases:
-
-            lex = lexer.Lexer(t['text'])
-            tokens = list(lex)
-            result = t['result']
-            assert(tokens == result)
-
-    def test_fail(self):
-        text = r'"should fail with missing closing quote'
-        lex = lexer.Lexer(text)
-        with pytest.raises(ValueError, match="No closing quotation"):
-            assert list(lex)
-
-    def test_stringio_text(self):
-        text = io.StringIO(r'Increase test coverage')
-        lex = lexer.Lexer(text)
-        tokens = list(lex)
-        result = ['Increase', 'test', 'coverage']
-        assert(tokens == result)
diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py
index 81e007f0..798ca5fe 100644
--- a/test/mitmproxy/tools/console/test_commander.py
+++ b/test/mitmproxy/tools/console/test_commander.py
@@ -165,8 +165,3 @@ class TestCommandBuffer:
             cb = commander.CommandBuffer(tctx.master)
             cb.text = "foo"
             assert cb.render()
-
-    def test_flatten(self):
-        with taddons.context() as tctx:
-            cb = commander.CommandBuffer(tctx.master)
-            assert cb.flatten("foo  bar") == "foo bar"
diff --git a/test/mitmproxy/tools/console/test_defaultkeys.py b/test/mitmproxy/tools/console/test_defaultkeys.py
index 035f71f7..7e8df6b6 100644
--- a/test/mitmproxy/tools/console/test_defaultkeys.py
+++ b/test/mitmproxy/tools/console/test_defaultkeys.py
@@ -3,12 +3,14 @@ from mitmproxy.tools.console import defaultkeys
 from mitmproxy.tools.console import keymap
 from mitmproxy.tools.console import master
 from mitmproxy import command
-
+from mitmproxy import ctx
 import pytest
 
 
 @pytest.mark.asyncio
 async def test_commands_exist():
+    command_manager = command.CommandManager(ctx)
+
     km = keymap.Keymap(None)
     defaultkeys.map(km)
     assert km.bindings
@@ -16,7 +18,10 @@ async def test_commands_exist():
     await m.load_flow(tflow())
 
     for binding in km.bindings:
-        cmd, *args = command.get_lexer(binding.command)
+        results = command_manager.parse_partial(binding.command)
+
+        cmd = results[0][0].value
+        args = [a.value for a in results[0][1:]]
 
         assert cmd in m.commands.commands
 
@@ -24,4 +29,7 @@ async def test_commands_exist():
         try:
             cmd_obj.prepare_args(args)
         except Exception as e:
+
+            import pdb
+            pdb.set_trace()
             raise ValueError("Invalid command: {}".format(binding.command)) from e