From 04e19f91716b9de6ec26df1478146eaedd47a329 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@corte.si>
Date: Thu, 14 Dec 2017 12:24:46 +1300
Subject: Introduce a custom widget for command editing

The builtin urwid.Edit widget is not sufficiently flexible for what we want to
do.
---
 mitmproxy/tools/console/commandeditor.py       |  9 ---
 mitmproxy/tools/console/commander/__init__.py  |  1 +
 mitmproxy/tools/console/commander/commander.py | 85 ++++++++++++++++++++++++++
 mitmproxy/tools/console/statusbar.py           |  5 +-
 test/mitmproxy/tools/console/test_commander.py | 37 +++++++++++
 5 files changed, 126 insertions(+), 11 deletions(-)
 create mode 100644 mitmproxy/tools/console/commander/__init__.py
 create mode 100644 mitmproxy/tools/console/commander/commander.py
 create mode 100644 test/mitmproxy/tools/console/test_commander.py

diff --git a/mitmproxy/tools/console/commandeditor.py b/mitmproxy/tools/console/commandeditor.py
index 17d1506b..e57ddbb4 100644
--- a/mitmproxy/tools/console/commandeditor.py
+++ b/mitmproxy/tools/console/commandeditor.py
@@ -1,19 +1,10 @@
 import typing
-import urwid
 
 from mitmproxy import exceptions
 from mitmproxy import flow
 from mitmproxy.tools.console import signals
 
 
-class CommandEdit(urwid.Edit):
-    def __init__(self, partial):
-        urwid.Edit.__init__(self, ":", partial)
-
-    def keypress(self, size, key):
-        return urwid.Edit.keypress(self, size, key)
-
-
 class CommandExecutor:
     def __init__(self, master):
         self.master = master
diff --git a/mitmproxy/tools/console/commander/__init__.py b/mitmproxy/tools/console/commander/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/mitmproxy/tools/console/commander/__init__.py
@@ -0,0 +1 @@
+
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
new file mode 100644
index 00000000..74855e4d
--- /dev/null
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -0,0 +1,85 @@
+import urwid
+from urwid.text_layout import calc_coords
+
+
+class CommandBuffer():
+    def __init__(self, start: str = ""):
+        self.buf = start
+        # This is the logical cursor position - the display cursor is one
+        # character further on. Cursor is always within the range [0:len(buffer)].
+        self._cursor = len(self.buf)
+
+    @property
+    def cursor(self):
+        return self._cursor
+
+    @cursor.setter
+    def cursor(self, x):
+        if x < 0:
+            self._cursor = 0
+        elif x > len(self.buf):
+            self._cursor = len(self.buf)
+        else:
+            self._cursor = x
+
+    def render(self):
+        return self.buf
+
+    def left(self):
+        self.cursor = self.cursor - 1
+
+    def right(self):
+        self.cursor = self.cursor + 1
+
+    def backspace(self):
+        if self.cursor == 0:
+            return
+        self.buf = self.buf[:self.cursor - 1] + self.buf[self.cursor:]
+        self.cursor = self.cursor - 1
+
+    def insert(self, k: str):
+        """
+            Inserts text at the cursor.
+        """
+        self.buf = self.buf = self.buf[:self.cursor] + k + self.buf[self.cursor:]
+        self.cursor += 1
+
+
+class CommandEdit(urwid.WidgetWrap):
+    leader = ": "
+
+    def __init__(self, text):
+        self.cbuf = CommandBuffer(text)
+        self._w = urwid.Text(self.leader)
+        self.update()
+
+    def keypress(self, size, key):
+        if key == "backspace":
+            self.cbuf.backspace()
+        elif key == "left":
+            self.cbuf.left()
+        elif key == "right":
+            self.cbuf.right()
+        elif len(key) == 1:
+            self.cbuf.insert(key)
+        self.update()
+
+    def update(self):
+        self._w.set_text([self.leader, self.cbuf.render()])
+
+    def render(self, size, focus=False):
+        (maxcol,) = size
+        canv = self._w.render((maxcol,))
+        canv = urwid.CompositeCanvas(canv)
+        canv.cursor = self.get_cursor_coords((maxcol,))
+        return canv
+
+    def get_cursor_coords(self, size):
+        p = self.cbuf.cursor + len(self.leader)
+        trans = self._w.get_line_translation(size[0])
+        x, y = calc_coords(self._w.get_text()[0], trans, p)
+        return x, y
+
+    def get_value(self):
+        return self.cbuf.buf
+
diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py
index 795b3d8a..a59fc92e 100644
--- a/mitmproxy/tools/console/statusbar.py
+++ b/mitmproxy/tools/console/statusbar.py
@@ -6,6 +6,7 @@ from mitmproxy.tools.console import common
 from mitmproxy.tools.console import signals
 from mitmproxy.tools.console import commandeditor
 import mitmproxy.tools.console.master # noqa
+from mitmproxy.tools.console.commander import commander
 
 
 class PromptPath:
@@ -66,7 +67,7 @@ class ActionBar(urwid.WidgetWrap):
 
     def sig_prompt_command(self, sender, partial=""):
         signals.focus.send(self, section="footer")
-        self._w = commandeditor.CommandEdit(partial)
+        self._w = commander.CommandEdit(partial)
         self.prompting = commandeditor.CommandExecutor(self.master)
 
     def sig_prompt_onekey(self, sender, prompt, keys, callback, args=()):
@@ -100,7 +101,7 @@ class ActionBar(urwid.WidgetWrap):
                 elif k in self.onekey:
                     self.prompt_execute(k)
             elif k == "enter":
-                self.prompt_execute(self._w.get_edit_text())
+                self.prompt_execute(self._w.get_value())
             else:
                 if common.is_keypress(k):
                     self._w.keypress(size, k)
diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py
new file mode 100644
index 00000000..b1f23df4
--- /dev/null
+++ b/test/mitmproxy/tools/console/test_commander.py
@@ -0,0 +1,37 @@
+
+from mitmproxy.tools.console.commander import commander
+
+
+class TestCommandBuffer:
+
+    def test_backspace(self):
+        tests = [
+            [("", 0), ("", 0)],
+            [("1", 0), ("1", 0)],
+            [("1", 1), ("", 0)],
+            [("123", 3), ("12", 2)],
+            [("123", 2), ("13", 1)],
+            [("123", 0), ("123", 0)],
+        ]
+        for start, output in tests:
+            cb = commander.CommandBuffer()
+            cb.buf, cb.cursor = start[0], start[1]
+            cb.backspace()
+            assert cb.buf == output[0]
+            assert cb.cursor == output[1]
+
+    def test_insert(self):
+        tests = [
+            [("", 0), ("x", 1)],
+            [("a", 0), ("xa", 1)],
+            [("xa", 2), ("xax", 3)],
+        ]
+        for start, output in tests:
+            cb = commander.CommandBuffer()
+            cb.buf, cb.cursor = start[0], start[1]
+            cb.insert("x")
+            assert cb.buf == output[0]
+            assert cb.cursor == output[1]
+
+
+
-- 
cgit v1.2.3


From e64d5c6bb963dedf291ed3c694ef735c8980a019 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Thu, 14 Dec 2017 15:43:15 +1300
Subject: commands: add a Cmd argument type

This represents a command passed as an argument. Also split arguments from
command values themselves, making the command help for meta-commands much
clearer.
---
 mitmproxy/addons/clientplayback.py             |  2 +-
 mitmproxy/addons/serverplayback.py             |  2 +-
 mitmproxy/command.py                           |  6 ++++++
 mitmproxy/tools/console/commander/__init__.py  |  1 -
 mitmproxy/tools/console/commander/commander.py | 17 ++++++++---------
 mitmproxy/tools/console/consoleaddons.py       | 20 +++++++++++++++-----
 test/mitmproxy/test_command.py                 |  4 ++++
 test/mitmproxy/tools/console/test_commander.py |  3 ---
 8 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py
index 9e012b67..fcc3209b 100644
--- a/mitmproxy/addons/clientplayback.py
+++ b/mitmproxy/addons/clientplayback.py
@@ -37,7 +37,7 @@ class ClientPlayback:
         ctx.master.addons.trigger("update", [])
 
     @command.command("replay.client.file")
-    def load_file(self, path: str) -> None:
+    def load_file(self, path: command.Path) -> None:
         try:
             flows = io.read_flows_from_paths([path])
         except exceptions.FlowReadException as e:
diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py
index 927f6e15..46968a8d 100644
--- a/mitmproxy/addons/serverplayback.py
+++ b/mitmproxy/addons/serverplayback.py
@@ -31,7 +31,7 @@ class ServerPlayback:
         ctx.master.addons.trigger("update", [])
 
     @command.command("replay.server.file")
-    def load_file(self, path: str) -> None:
+    def load_file(self, path: command.Path) -> None:
         try:
             flows = io.read_flows_from_paths([path])
         except exceptions.FlowReadException as e:
diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index c4821973..a638b7ee 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -23,6 +23,10 @@ class Path(str):
     pass
 
 
+class Cmd(str):
+    pass
+
+
 def typename(t: type, ret: bool) -> str:
     """
         Translates a type to an explanatory string. If ret is True, we're
@@ -172,6 +176,8 @@ def parsearg(manager: CommandManager, spec: str, argtype: type) -> typing.Any:
                 "Invalid choice: see %s for options" % cmd
             )
         return spec
+    if argtype in (Path, Cmd):
+        return spec
     elif issubclass(argtype, str):
         return spec
     elif argtype == bool:
diff --git a/mitmproxy/tools/console/commander/__init__.py b/mitmproxy/tools/console/commander/__init__.py
index 8b137891..e69de29b 100644
--- a/mitmproxy/tools/console/commander/__init__.py
+++ b/mitmproxy/tools/console/commander/__init__.py
@@ -1 +0,0 @@
-
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index 74855e4d..d07910a6 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -3,18 +3,18 @@ from urwid.text_layout import calc_coords
 
 
 class CommandBuffer():
-    def __init__(self, start: str = ""):
+    def __init__(self, start: str = "") -> None:
         self.buf = start
         # This is the logical cursor position - the display cursor is one
         # character further on. Cursor is always within the range [0:len(buffer)].
         self._cursor = len(self.buf)
 
     @property
-    def cursor(self):
+    def cursor(self) -> int:
         return self._cursor
 
     @cursor.setter
-    def cursor(self, x):
+    def cursor(self, x) -> None:
         if x < 0:
             self._cursor = 0
         elif x > len(self.buf):
@@ -25,19 +25,19 @@ class CommandBuffer():
     def render(self):
         return self.buf
 
-    def left(self):
+    def left(self) -> None:
         self.cursor = self.cursor - 1
 
-    def right(self):
+    def right(self) -> None:
         self.cursor = self.cursor + 1
 
-    def backspace(self):
+    def backspace(self) -> None:
         if self.cursor == 0:
             return
         self.buf = self.buf[:self.cursor - 1] + self.buf[self.cursor:]
         self.cursor = self.cursor - 1
 
-    def insert(self, k: str):
+    def insert(self, k: str) -> None:
         """
             Inserts text at the cursor.
         """
@@ -48,7 +48,7 @@ class CommandBuffer():
 class CommandEdit(urwid.WidgetWrap):
     leader = ": "
 
-    def __init__(self, text):
+    def __init__(self, text) -> None:
         self.cbuf = CommandBuffer(text)
         self._w = urwid.Text(self.leader)
         self.update()
@@ -82,4 +82,3 @@ class CommandEdit(urwid.WidgetWrap):
 
     def get_value(self):
         return self.cbuf.buf
-
diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py
index 06ee3341..d79e3947 100644
--- a/mitmproxy/tools/console/consoleaddons.py
+++ b/mitmproxy/tools/console/consoleaddons.py
@@ -224,7 +224,11 @@ class ConsoleAddon:
 
     @command.command("console.choose")
     def console_choose(
-        self, prompt: str, choices: typing.Sequence[str], *cmd: str
+        self,
+        prompt: str,
+        choices: typing.Sequence[str],
+        cmd: command.Cmd,
+        *args: str,
     ) -> None:
         """
             Prompt the user to choose from a specified list of strings, then
@@ -233,7 +237,7 @@ class ConsoleAddon:
         """
         def callback(opt):
             # We're now outside of the call context...
-            repl = " ".join(cmd)
+            repl = cmd + " " + " ".join(args)
             repl = repl.replace("{choice}", opt)
             try:
                 self.master.commands.call(repl)
@@ -246,7 +250,7 @@ class ConsoleAddon:
 
     @command.command("console.choose.cmd")
     def console_choose_cmd(
-        self, prompt: str, choicecmd: str, *cmd: str
+        self, prompt: str, choicecmd: command.Cmd, *cmd: str
     ) -> None:
         """
             Prompt the user to choose from a list of strings returned by a
@@ -492,14 +496,20 @@ class ConsoleAddon:
         return list(sorted(keymap.Contexts))
 
     @command.command("console.key.bind")
-    def key_bind(self, contexts: typing.Sequence[str], key: str, *command: str) -> None:
+    def key_bind(
+        self,
+        contexts: typing.Sequence[str],
+        key: str,
+        cmd: command.Cmd,
+        *args: str,
+    ) -> None:
         """
             Bind a shortcut key.
         """
         try:
             self.master.keymap.add(
                 key,
-                " ".join(command),
+                cmd + " " + " ".join(args),
                 contexts,
                 ""
             )
diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py
index e1879ba2..c8007463 100644
--- a/test/mitmproxy/test_command.py
+++ b/test/mitmproxy/test_command.py
@@ -100,6 +100,7 @@ def test_typename():
 
     assert command.typename(command.Choice("foo"), False) == "choice"
     assert command.typename(command.Path, False) == "path"
+    assert command.typename(command.Cmd, False) == "cmd"
 
 
 class DummyConsole:
@@ -162,6 +163,9 @@ def test_parsearg():
         assert command.parsearg(
             tctx.master.commands, "foo", command.Path
         ) == "foo"
+        assert command.parsearg(
+            tctx.master.commands, "foo", command.Cmd
+        ) == "foo"
 
 
 class TDec:
diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py
index b1f23df4..fdf54897 100644
--- a/test/mitmproxy/tools/console/test_commander.py
+++ b/test/mitmproxy/tools/console/test_commander.py
@@ -32,6 +32,3 @@ class TestCommandBuffer:
             cb.insert("x")
             assert cb.buf == output[0]
             assert cb.cursor == output[1]
-
-
-
-- 
cgit v1.2.3


From 0cd4a7726892ecda494d94bd12af0094c53a6a85 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Thu, 14 Dec 2017 17:49:53 +1300
Subject: commands: add a parser for partial commands

We only return Cmd and str types for the moment.
---
 mitmproxy/command.py           | 25 +++++++++++++++++++++++++
 test/mitmproxy/test_command.py | 10 ++++++++++
 2 files changed, 35 insertions(+)

diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index a638b7ee..f73eeb68 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -3,6 +3,7 @@
 """
 import inspect
 import types
+import io
 import typing
 import shlex
 import textwrap
@@ -137,6 +138,30 @@ class CommandManager:
     def add(self, path: str, func: typing.Callable):
         self.commands[path] = Command(self, path, func)
 
+    def parse_partial(self, cmdstr: str) -> typing.Sequence[typing.Tuple[str, type]]:
+        """
+            Parse a possibly partial command. Return a sequence of (part, type) tuples.
+        """
+        parts: typing.List[typing.Tuple[str, type]] = []
+        buf = io.StringIO(cmdstr)
+        # mypy mis-identifies shlex.shlex as abstract
+        lex = shlex.shlex(buf)  # type: ignore
+        while 1:
+            remainder = cmdstr[buf.tell():]
+            try:
+                t = lex.get_token()
+            except ValueError:
+                parts.append((remainder, str))
+                break
+            if not t:
+                break
+            typ: type = str
+            # First value is a special case: it has to be a command
+            if not parts:
+                typ = Cmd
+            parts.append((t, typ))
+        return parts
+
     def call_args(self, path, args):
         """
             Call a command using a list of string arguments. May raise CommandError.
diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py
index c8007463..5218042c 100644
--- a/test/mitmproxy/test_command.py
+++ b/test/mitmproxy/test_command.py
@@ -64,6 +64,16 @@ class TestCommand:
             c = command.Command(cm, "cmd.three", a.cmd3)
             assert c.call(["1"]) == 1
 
+    def test_parse_partial(self):
+        tests = [
+            ["foo bar", [("foo", command.Cmd), ("bar", str)]],
+            ["foo 'bar", [("foo", command.Cmd), ("'bar", str)]],
+        ]
+        with taddons.context() as tctx:
+            cm = command.CommandManager(tctx.master)
+            for s, expected in tests:
+                assert cm.parse_partial(s) == expected
+
 
 def test_simple():
     with taddons.context() as tctx:
-- 
cgit v1.2.3


From 4d358c49fbeafe504cc7b9a8b66ea572c8cbb0ee Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Fri, 15 Dec 2017 07:20:07 +1300
Subject: WIP: autocompletion

---
 mitmproxy/command.py                           | 15 ++++++++---
 mitmproxy/tools/console/commander/commander.py | 25 +++++++++++++++---
 mitmproxy/tools/console/statusbar.py           |  2 +-
 test/mitmproxy/test_command.py                 | 18 +++++++++++--
 test/mitmproxy/tools/console/test_commander.py | 35 ++++++++++++++++----------
 5 files changed, 72 insertions(+), 23 deletions(-)

diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index f73eeb68..aab721b5 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -123,6 +123,11 @@ class Command:
         return ret
 
 
+class ParseResult(typing.NamedTuple):
+    value: str
+    type: type
+
+
 class CommandManager:
     def __init__(self, master):
         self.master = master
@@ -138,11 +143,11 @@ class CommandManager:
     def add(self, path: str, func: typing.Callable):
         self.commands[path] = Command(self, path, func)
 
-    def parse_partial(self, cmdstr: str) -> typing.Sequence[typing.Tuple[str, type]]:
+    def parse_partial(self, cmdstr: str) -> typing.Sequence[ParseResult]:
         """
             Parse a possibly partial command. Return a sequence of (part, type) tuples.
         """
-        parts: typing.List[typing.Tuple[str, type]] = []
+        parts: typing.List[ParseResult] = []
         buf = io.StringIO(cmdstr)
         # mypy mis-identifies shlex.shlex as abstract
         lex = shlex.shlex(buf)  # type: ignore
@@ -151,7 +156,7 @@ class CommandManager:
             try:
                 t = lex.get_token()
             except ValueError:
-                parts.append((remainder, str))
+                parts.append(ParseResult(value = remainder, type = str))
                 break
             if not t:
                 break
@@ -159,7 +164,9 @@ class CommandManager:
             # First value is a special case: it has to be a command
             if not parts:
                 typ = Cmd
-            parts.append((t, typ))
+            parts.append(ParseResult(value = t, type = typ))
+        if not parts:
+            return [ParseResult(value = "", type = Cmd)]
         return parts
 
     def call_args(self, path, args):
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index d07910a6..d961d421 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -1,9 +1,19 @@
 import urwid
 from urwid.text_layout import calc_coords
+import typing
+
+import mitmproxy.master
+import mitmproxy.command
+
+
+class CompletionState:
+    def __init__(self, parts: typing.Sequence[mitmproxy.command.ParseResult]) -> None:
+        self.parts = parts
 
 
 class CommandBuffer():
-    def __init__(self, start: str = "") -> None:
+    def __init__(self, master: mitmproxy.master.Master, start: str = "") -> None:
+        self.master = master
         self.buf = start
         # This is the logical cursor position - the display cursor is one
         # character further on. Cursor is always within the range [0:len(buffer)].
@@ -31,6 +41,12 @@ class CommandBuffer():
     def right(self) -> None:
         self.cursor = self.cursor + 1
 
+    def cycle_completion(self) -> None:
+        parts = self.master.commands.parse_partial(self.buf[:self.cursor])
+        if parts[-1][1] == str:
+            return
+        raise ValueError
+
     def backspace(self) -> None:
         if self.cursor == 0:
             return
@@ -48,8 +64,9 @@ class CommandBuffer():
 class CommandEdit(urwid.WidgetWrap):
     leader = ": "
 
-    def __init__(self, text) -> None:
-        self.cbuf = CommandBuffer(text)
+    def __init__(self, master: mitmproxy.master.Master, text: str) -> None:
+        self.master = master
+        self.cbuf = CommandBuffer(master, text)
         self._w = urwid.Text(self.leader)
         self.update()
 
@@ -60,6 +77,8 @@ class CommandEdit(urwid.WidgetWrap):
             self.cbuf.left()
         elif key == "right":
             self.cbuf.right()
+        elif key == "tab":
+            self.cbuf.cycle_completion()
         elif len(key) == 1:
             self.cbuf.insert(key)
         self.update()
diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py
index a59fc92e..6a1f07a9 100644
--- a/mitmproxy/tools/console/statusbar.py
+++ b/mitmproxy/tools/console/statusbar.py
@@ -67,7 +67,7 @@ class ActionBar(urwid.WidgetWrap):
 
     def sig_prompt_command(self, sender, partial=""):
         signals.focus.send(self, section="footer")
-        self._w = commander.CommandEdit(partial)
+        self._w = commander.CommandEdit(self.master, partial)
         self.prompting = commandeditor.CommandExecutor(self.master)
 
     def sig_prompt_onekey(self, sender, prompt, keys, callback, args=()):
diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py
index 5218042c..b4711236 100644
--- a/test/mitmproxy/test_command.py
+++ b/test/mitmproxy/test_command.py
@@ -66,8 +66,22 @@ class TestCommand:
 
     def test_parse_partial(self):
         tests = [
-            ["foo bar", [("foo", command.Cmd), ("bar", str)]],
-            ["foo 'bar", [("foo", command.Cmd), ("'bar", str)]],
+            [
+                "foo bar",
+                [
+                    command.ParseResult(value = "foo", type = command.Cmd),
+                    command.ParseResult(value = "bar", type = str)
+                ],
+            ],
+            [
+                "foo 'bar",
+                [
+                    command.ParseResult(value = "foo", type = command.Cmd),
+                    command.ParseResult(value = "'bar", type = str)
+                ]
+            ],
+            ["a", [command.ParseResult(value = "a", type = command.Cmd)]],
+            ["", [command.ParseResult(value = "", type = command.Cmd)]],
         ]
         with taddons.context() as tctx:
             cm = command.CommandManager(tctx.master)
diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py
index fdf54897..9ef4a318 100644
--- a/test/mitmproxy/tools/console/test_commander.py
+++ b/test/mitmproxy/tools/console/test_commander.py
@@ -1,5 +1,5 @@
-
 from mitmproxy.tools.console.commander import commander
+from mitmproxy.test import taddons
 
 
 class TestCommandBuffer:
@@ -13,12 +13,13 @@ class TestCommandBuffer:
             [("123", 2), ("13", 1)],
             [("123", 0), ("123", 0)],
         ]
-        for start, output in tests:
-            cb = commander.CommandBuffer()
-            cb.buf, cb.cursor = start[0], start[1]
-            cb.backspace()
-            assert cb.buf == output[0]
-            assert cb.cursor == output[1]
+        with taddons.context() as tctx:
+            for start, output in tests:
+                cb = commander.CommandBuffer(tctx.master)
+                cb.buf, cb.cursor = start[0], start[1]
+                cb.backspace()
+                assert cb.buf == output[0]
+                assert cb.cursor == output[1]
 
     def test_insert(self):
         tests = [
@@ -26,9 +27,17 @@ class TestCommandBuffer:
             [("a", 0), ("xa", 1)],
             [("xa", 2), ("xax", 3)],
         ]
-        for start, output in tests:
-            cb = commander.CommandBuffer()
-            cb.buf, cb.cursor = start[0], start[1]
-            cb.insert("x")
-            assert cb.buf == output[0]
-            assert cb.cursor == output[1]
+        with taddons.context() as tctx:
+            for start, output in tests:
+                cb = commander.CommandBuffer(tctx.master)
+                cb.buf, cb.cursor = start[0], start[1]
+                cb.insert("x")
+                assert cb.buf == output[0]
+                assert cb.cursor == output[1]
+
+    def test_cycle_completion(self):
+        with taddons.context() as tctx:
+            cb = commander.CommandBuffer(tctx.master)
+            cb.buf = "foo bar"
+            cb.cursor = len(cb.buf)
+            cb.cycle_completion()
-- 
cgit v1.2.3


From 8c0ba71fd8f2e29e8348c88aa6d653d3161ea20a Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@corte.si>
Date: Fri, 15 Dec 2017 09:43:09 +1300
Subject: commander: tab completion for command names

---
 mitmproxy/command.py                           |  2 +-
 mitmproxy/tools/console/commander/commander.py | 60 ++++++++++++++++++++++----
 test/mitmproxy/tools/console/test_commander.py | 25 +++++++++++
 3 files changed, 77 insertions(+), 10 deletions(-)

diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index aab721b5..b909dfd5 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -125,7 +125,7 @@ class Command:
 
 class ParseResult(typing.NamedTuple):
     value: str
-    type: type
+    type: typing.Type
 
 
 class CommandManager:
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index d961d421..a0c3a3b2 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -1,23 +1,51 @@
 import urwid
 from urwid.text_layout import calc_coords
 import typing
+import abc
 
 import mitmproxy.master
 import mitmproxy.command
 
 
-class CompletionState:
-    def __init__(self, parts: typing.Sequence[mitmproxy.command.ParseResult]) -> None:
-        self.parts = parts
+class Completer:
+    @abc.abstractmethod
+    def cycle(self) -> str:
+        pass
+
+
+class ListCompleter(Completer):
+    def __init__(
+        self,
+        start: str,
+        options: typing.Sequence[str],
+    ) -> None:
+        self.start = start
+        self.options = []  # type: typing.Sequence[str]
+        for o in options:
+            if o.startswith(start):
+                self.options.append(o)
+        self.offset = 0
+
+    def cycle(self) -> str:
+        if not self.options:
+            return self.start
+        ret = self.options[self.offset]
+        self.offset = (self.offset + 1) % len(self.options)
+        return ret
+
+
+class CompletionState(typing.NamedTuple):
+    completer: Completer
+    parse: typing.Sequence[mitmproxy.command.ParseResult]
 
 
 class CommandBuffer():
     def __init__(self, master: mitmproxy.master.Master, start: str = "") -> None:
         self.master = master
         self.buf = start
-        # This is the logical cursor position - the display cursor is one
-        # character further on. Cursor is always within the range [0:len(buffer)].
+        # Cursor is always within the range [0:len(buffer)].
         self._cursor = len(self.buf)
+        self.completion = None   # type: CompletionState
 
     @property
     def cursor(self) -> int:
@@ -42,16 +70,29 @@ class CommandBuffer():
         self.cursor = self.cursor + 1
 
     def cycle_completion(self) -> None:
-        parts = self.master.commands.parse_partial(self.buf[:self.cursor])
-        if parts[-1][1] == str:
-            return
-        raise ValueError
+        if not self.completion:
+            parts = self.master.commands.parse_partial(self.buf[:self.cursor])
+            if parts[-1].type == mitmproxy.command.Cmd:
+                self.completion = CompletionState(
+                    completer = ListCompleter(
+                        parts[-1].value,
+                        self.master.commands.commands.keys(),
+                    ),
+                    parse = parts,
+                )
+        if self.completion:
+            nxt = self.completion.completer.cycle()
+            buf = " ".join([i.value for i in self.completion.parse[:-1]]) + " " + nxt
+            buf = buf.strip()
+            self.buf = buf
+            self.cursor = len(self.buf)
 
     def backspace(self) -> None:
         if self.cursor == 0:
             return
         self.buf = self.buf[:self.cursor - 1] + self.buf[self.cursor:]
         self.cursor = self.cursor - 1
+        self.completion = None
 
     def insert(self, k: str) -> None:
         """
@@ -59,6 +100,7 @@ class CommandBuffer():
         """
         self.buf = self.buf = self.buf[:self.cursor] + k + self.buf[self.cursor:]
         self.cursor += 1
+        self.completion = None
 
 
 class CommandEdit(urwid.WidgetWrap):
diff --git a/test/mitmproxy/tools/console/test_commander.py b/test/mitmproxy/tools/console/test_commander.py
index 9ef4a318..1ac4c5c6 100644
--- a/test/mitmproxy/tools/console/test_commander.py
+++ b/test/mitmproxy/tools/console/test_commander.py
@@ -2,6 +2,31 @@ from mitmproxy.tools.console.commander import commander
 from mitmproxy.test import taddons
 
 
+class TestListCompleter:
+    def test_cycle(self):
+        tests = [
+            [
+                "",
+                ["a", "b", "c"],
+                ["a", "b", "c", "a"]
+            ],
+            [
+                "xxx",
+                ["a", "b", "c"],
+                ["xxx", "xxx", "xxx"]
+            ],
+            [
+                "b",
+                ["a", "b", "ba", "bb", "c"],
+                ["b", "ba", "bb", "b"]
+            ],
+        ]
+        for start, options, cycle in tests:
+            c = commander.ListCompleter(start, options)
+            for expected in cycle:
+                assert c.cycle() == expected
+
+
 class TestCommandBuffer:
 
     def test_backspace(self):
-- 
cgit v1.2.3


From 1c097813c16d530631562727cd9c2db0fae8755d Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@corte.si>
Date: Fri, 15 Dec 2017 11:34:53 +1300
Subject: commands: emit types from partial parser, implement choice completion

---
 mitmproxy/command.py                           | 51 +++++++++++++++++---------
 mitmproxy/tools/console/commander/commander.py | 11 +++++-
 test/mitmproxy/test_command.py                 | 24 +++++++++++-
 3 files changed, 66 insertions(+), 20 deletions(-)

diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index b909dfd5..7374a19a 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -15,6 +15,14 @@ from mitmproxy import exceptions
 from mitmproxy import flow
 
 
+def lexer(s):
+    # mypy mis-identifies shlex.shlex as abstract
+    lex = shlex.shlex(s, punctuation_chars=True)  # type: ignore
+    lex.whitespace_split = True
+    lex.commenters = ''
+    return lex
+
+
 Cuts = typing.Sequence[
     typing.Sequence[typing.Union[str, bytes]]
 ]
@@ -123,9 +131,10 @@ class Command:
         return ret
 
 
-class ParseResult(typing.NamedTuple):
-    value: str
-    type: typing.Type
+ParseResult = typing.NamedTuple(
+    "ParseResult",
+    [("value", str), ("type", typing.Type)],
+)
 
 
 class CommandManager:
@@ -147,27 +156,37 @@ class CommandManager:
         """
             Parse a possibly partial command. Return a sequence of (part, type) tuples.
         """
-        parts: typing.List[ParseResult] = []
         buf = io.StringIO(cmdstr)
-        # mypy mis-identifies shlex.shlex as abstract
-        lex = shlex.shlex(buf)  # type: ignore
+        parts: typing.List[str] = []
+        lex = lexer(buf)
         while 1:
             remainder = cmdstr[buf.tell():]
             try:
                 t = lex.get_token()
             except ValueError:
-                parts.append(ParseResult(value = remainder, type = str))
+                parts.append(remainder)
                 break
             if not t:
                 break
-            typ: type = str
-            # First value is a special case: it has to be a command
-            if not parts:
-                typ = Cmd
-            parts.append(ParseResult(value = t, type = typ))
+            parts.append(t)
         if not parts:
-            return [ParseResult(value = "", type = Cmd)]
-        return parts
+            parts = [""]
+        elif cmdstr.endswith(" "):
+            parts.append("")
+
+        parse: typing.List[ParseResult] = []
+        params: typing.List[type] = []
+        for i in range(len(parts)):
+            if i == 0:
+                params[:] = [Cmd]
+                if parts[i] in self.commands:
+                    params.extend(self.commands[parts[i]].paramtypes)
+            if params:
+                typ = params.pop(0)
+            else:
+                typ = str
+            parse.append(ParseResult(value=parts[i], type=typ))
+        return parse
 
     def call_args(self, path, args):
         """
@@ -181,7 +200,7 @@ class CommandManager:
         """
             Call a command using a string. May raise CommandError.
         """
-        parts = shlex.split(cmdstr)
+        parts = list(lexer(cmdstr))
         if not len(parts) >= 1:
             raise exceptions.CommandError("Invalid command: %s" % cmdstr)
         return self.call_args(parts[0], parts[1:])
@@ -208,8 +227,6 @@ def parsearg(manager: CommandManager, spec: str, argtype: type) -> typing.Any:
                 "Invalid choice: see %s for options" % cmd
             )
         return spec
-    if argtype in (Path, Cmd):
-        return spec
     elif issubclass(argtype, str):
         return spec
     elif argtype == bool:
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index a0c3a3b2..f82ce9ce 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -72,7 +72,8 @@ class CommandBuffer():
     def cycle_completion(self) -> None:
         if not self.completion:
             parts = self.master.commands.parse_partial(self.buf[:self.cursor])
-            if parts[-1].type == mitmproxy.command.Cmd:
+            last = parts[-1]
+            if last.type == mitmproxy.command.Cmd:
                 self.completion = CompletionState(
                     completer = ListCompleter(
                         parts[-1].value,
@@ -80,6 +81,14 @@ class CommandBuffer():
                     ),
                     parse = parts,
                 )
+            elif isinstance(last.type, mitmproxy.command.Choice):
+                self.completion = CompletionState(
+                    completer = ListCompleter(
+                        parts[-1].value,
+                        self.master.commands.call(last.type.options_command),
+                    ),
+                    parse = parts,
+                )
         if self.completion:
             nxt = self.completion.completer.cycle()
             buf = " ".join([i.value for i in self.completion.parse[:-1]]) + " " + nxt
diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py
index b4711236..76ce2245 100644
--- a/test/mitmproxy/test_command.py
+++ b/test/mitmproxy/test_command.py
@@ -11,19 +11,24 @@ from mitmproxy.utils import typecheck
 
 
 class TAddon:
+    @command.command("cmd1")
     def cmd1(self, foo: str) -> str:
         """cmd1 help"""
         return "ret " + foo
 
+    @command.command("cmd2")
     def cmd2(self, foo: str) -> str:
         return 99
 
+    @command.command("cmd3")
     def cmd3(self, foo: int) -> int:
         return foo
 
+    @command.command("empty")
     def empty(self) -> None:
         pass
 
+    @command.command("varargs")
     def varargs(self, one: str, *var: str) -> typing.Sequence[str]:
         return list(var)
 
@@ -34,6 +39,7 @@ class TAddon:
     def choose(self, arg: str) -> typing.Sequence[str]:
         return ["one", "two", "three"]
 
+    @command.command("path")
     def path(self, arg: command.Path) -> None:
         pass
 
@@ -82,11 +88,25 @@ class TestCommand:
             ],
             ["a", [command.ParseResult(value = "a", type = command.Cmd)]],
             ["", [command.ParseResult(value = "", type = command.Cmd)]],
+            [
+                "cmd3 1",
+                [
+                    command.ParseResult(value = "cmd3", type = command.Cmd),
+                    command.ParseResult(value = "1", type = int),
+                ]
+            ],
+            [
+                "cmd3 ",
+                [
+                    command.ParseResult(value = "cmd3", type = command.Cmd),
+                    command.ParseResult(value = "", type = int),
+                ]
+            ],
         ]
         with taddons.context() as tctx:
-            cm = command.CommandManager(tctx.master)
+            tctx.master.addons.add(TAddon())
             for s, expected in tests:
-                assert cm.parse_partial(s) == expected
+                assert tctx.master.commands.parse_partial(s) == expected
 
 
 def test_simple():
-- 
cgit v1.2.3


From 2cfe45428ae1df16d0f7a5cbb9a541a441b7413d Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@corte.si>
Date: Fri, 15 Dec 2017 11:42:26 +1300
Subject: command: add command.Arg type

This type represents an argument to a command passed to another command. This
improves help text, and will be used in the partial parser to expand subcommand
types.
---
 mitmproxy/command.py                           | 13 +++++++++----
 mitmproxy/tools/console/commander/commander.py | 10 +++++++---
 mitmproxy/tools/console/consoleaddons.py       |  6 +++---
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index 7374a19a..087f7770 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -17,7 +17,8 @@ from mitmproxy import flow
 
 def lexer(s):
     # mypy mis-identifies shlex.shlex as abstract
-    lex = shlex.shlex(s, punctuation_chars=True)  # type: ignore
+    lex = shlex.shlex(s)  # type: ignore
+    lex.wordchars += "."
     lex.whitespace_split = True
     lex.commenters = ''
     return lex
@@ -36,6 +37,10 @@ class Cmd(str):
     pass
 
 
+class Arg(str):
+    pass
+
+
 def typename(t: type, ret: bool) -> str:
     """
         Translates a type to an explanatory string. If ret is True, we're
@@ -157,7 +162,7 @@ class CommandManager:
             Parse a possibly partial command. Return a sequence of (part, type) tuples.
         """
         buf = io.StringIO(cmdstr)
-        parts: typing.List[str] = []
+        parts = []  # type: typing.List[str]
         lex = lexer(buf)
         while 1:
             remainder = cmdstr[buf.tell():]
@@ -174,8 +179,8 @@ class CommandManager:
         elif cmdstr.endswith(" "):
             parts.append("")
 
-        parse: typing.List[ParseResult] = []
-        params: typing.List[type] = []
+        parse = []  # type: typing.List[ParseResult]
+        params = []  # type: typing.List[type]
         for i in range(len(parts)):
             if i == 0:
                 params[:] = [Cmd]
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index f82ce9ce..dbbc8ff2 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -34,9 +34,13 @@ class ListCompleter(Completer):
         return ret
 
 
-class CompletionState(typing.NamedTuple):
-    completer: Completer
-    parse: typing.Sequence[mitmproxy.command.ParseResult]
+CompletionState = typing.NamedTuple(
+    "CompletionState",
+    [
+        ("completer", Completer),
+        ("parse", typing.Sequence[mitmproxy.command.ParseResult])
+    ]
+)
 
 
 class CommandBuffer():
diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py
index d79e3947..023cc5d9 100644
--- a/mitmproxy/tools/console/consoleaddons.py
+++ b/mitmproxy/tools/console/consoleaddons.py
@@ -228,7 +228,7 @@ class ConsoleAddon:
         prompt: str,
         choices: typing.Sequence[str],
         cmd: command.Cmd,
-        *args: str,
+        *args: command.Arg
     ) -> None:
         """
             Prompt the user to choose from a specified list of strings, then
@@ -250,7 +250,7 @@ class ConsoleAddon:
 
     @command.command("console.choose.cmd")
     def console_choose_cmd(
-        self, prompt: str, choicecmd: command.Cmd, *cmd: str
+        self, prompt: str, choicecmd: command.Cmd, *cmd: command.Arg
     ) -> None:
         """
             Prompt the user to choose from a list of strings returned by a
@@ -501,7 +501,7 @@ class ConsoleAddon:
         contexts: typing.Sequence[str],
         key: str,
         cmd: command.Cmd,
-        *args: str,
+        *args: command.Arg
     ) -> None:
         """
             Bind a shortcut key.
-- 
cgit v1.2.3