Options tweaks

- Regularise presentation and wording
- Help is mandatory for all options
- Auto-generate wording to say that sequence options can be passed multiple
times on the command-line

4 files changed, 131 insertions, 111 deletions

diff --git a/mitmproxy/options.py b/mitmproxy/options.py
index 1b5b700a..21ee257d 100644
--- a/mitmproxy/options.py
+++ b/mitmproxy/options.py
@@ -55,20 +55,20 @@ class Options(optmanager.OptManager):
         self.add_option(
             "onboarding_host", APP_HOST, str,
             """
-                Domain to serve the onboarding app from. For transparent mode, use
-                an IP when a DNS entry for the app domain is not present. Default:
-                %s
+            Domain to serve the onboarding app from. For transparent mode, use
+            an IP when a DNS entry for the app domain is not present. Default:
+            %s
             """ % APP_HOST
         )
         self.add_option(
             "onboarding_port", APP_PORT, int,
-            help="Port to serve the onboarding app from."
+            "Port to serve the onboarding app from."
         )
         self.add_option(
             "anticache", False, bool,
             """
-                Strip out request headers that might cause the server to return
-                304-not-modified.
+            Strip out request headers that might cause the server to return
+            304-not-modified.
             """
         )
         self.add_option(
@@ -93,11 +93,17 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "server_replay_nopop", False, bool,
-            "Disable response pop from response flow. "
-            "This makes it possible to replay same response multiple times."
+            """
+            Disable response pop from response flow. This makes it possible to
+            replay same response multiple times.
+            """
         )
         self.add_option(
             "refresh_server_playback", True, bool,
+            """
+            Refresh server replay responses by adjusting date, expires and
+            last-modified headers, as well as adjusting cookie expiration.
+            """
         )
         self.add_option(
             "rfile", None, Optional[str],
@@ -106,8 +112,7 @@ class Options(optmanager.OptManager):
         self.add_option(
             "scripts", [], Sequence[str],
             """
-                Run a script. Surround with quotes to pass script arguments. Can
-                be passed multiple times.
+            Execute a script.
             """
         )
         self.add_option(
@@ -116,23 +121,28 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "replacements", [], Sequence[str],
-            "Replacement patterns."
+            """
+            Replacement patterns of the form "/pattern/regex/replacement", where
+            the separator can be any character.
+            """
         )
         self.add_option(
             "replacement_files", [], Sequence[str],
             """
-                Replacement pattern, where the replacement clause is a path to a
-                file.
+            Replacement pattern, where the replacement clause is a path to a
+            file.
             """
         )
         self.add_option(
             "server_replay_use_headers", [], Sequence[str],
-            "Request headers to be considered during replay. "
-            "Can be passed multiple times."
+            "Request headers to be considered during replay."
         )
         self.add_option(
             "setheaders", [], Sequence[str],
-            help="Header set pattern."
+            """
+            Header set pattern of the form "/pattern/header/value", where the
+            separator can be any character.
+            """
         )
         self.add_option(
             "server_replay", [], Sequence[str],
@@ -149,19 +159,22 @@ class Options(optmanager.OptManager):
         self.add_option(
             "stream_large_bodies", None, Optional[str],
             """
-                Stream data to the client if response body exceeds the given
-                threshold. If streamed, the body will not be stored in any way.
-                Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
+            Stream data to the client if response body exceeds the given
+            threshold. If streamed, the body will not be stored in any way.
+            Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
             """
         )
         self.add_option(
             "verbosity", 2, int,
             "Log verbosity."
         )
-        self.add_option("default_contentview", "auto", str)
+        self.add_option(
+            "default_contentview", "auto", str,
+            "The default content view mode."
+        )
         self.add_option(
             "streamfile", None, Optional[str],
-            help="Write flows to file. Prefix path with + to append."
+            "Write flows to file. Prefix path with + to append."
         )
         self.add_option(
             "server_replay_ignore_content", False, bool,
@@ -170,22 +183,24 @@ class Options(optmanager.OptManager):
         self.add_option(
             "server_replay_ignore_params", [], Sequence[str],
             """
-                Request's parameters to be ignored while searching for a saved
-                flow to replay. Can be passed multiple times.
+            Request's parameters to be ignored while searching for a saved flow
+            to replay. Can be passed multiple times.
             """
         )
         self.add_option(
             "server_replay_ignore_payload_params", [], Sequence[str],
             """
-                Request's payload parameters (application/x-www-form-urlencoded
-                or multipart/form-data) to be ignored while searching for a
-                saved flow to replay. Can be passed multiple times.
+            Request's payload parameters (application/x-www-form-urlencoded or
+            multipart/form-data) to be ignored while searching for a saved flow
+            to replay.
             """
         )
         self.add_option(
             "server_replay_ignore_host", False, bool,
-            "Ignore request's destination host while searching for a saved"
-            " flow to replay"
+            """
+            Ignore request's destination host while searching for a saved flow
+            to replay.
+            """
         )
 
         # Proxy options
@@ -196,8 +211,8 @@ class Options(optmanager.OptManager):
         self.add_option(
             "auth_singleuser", None, Optional[str],
             """
-                Allows access to a a single user, specified in the form
-                username:password.
+            Allows access to a a single user, specified in the form
+            username:password.
             """
         )
         self.add_option(
@@ -206,13 +221,17 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "add_upstream_certs_to_client_chain", False, bool,
-            "Add all certificates of the upstream server to the certificate chain "
-            "that will be served to the proxy client, as extras."
+            """
+            Add all certificates of the upstream server to the certificate chain
+            that will be served to the proxy client, as extras.
+            """
         )
         self.add_option(
             "body_size_limit", None, Optional[str],
-            "Byte size limit of HTTP request and response bodies."
-            " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
+            """
+            Byte size limit of HTTP request and response bodies. Understands
+            k/m/g suffixes, i.e. 3m for 3 megabytes.
+            """
         )
         self.add_option(
             "cadir", CA_DIR, str,
@@ -221,7 +240,7 @@ class Options(optmanager.OptManager):
         self.add_option(
             "certs", [], Sequence[str],
             """
-            Add an SSL certificate. SPEC is of the form "[domain=]path". The
+            SSL certificates. SPEC is of the form "[domain=]path". The
             domain may include a wildcard, and is equal to "*" if not specified.
             The file at path is a certificate in PEM format. If a private key is
             included in the PEM, it is used, else the default key in the conf
@@ -245,11 +264,11 @@ class Options(optmanager.OptManager):
         self.add_option(
             "ignore_hosts", [], Sequence[str],
             """
-                Ignore host and forward all traffic without processing it. In
-                transparent mode, it is recommended to use an IP address (range),
-                not the hostname. In regular mode, only SSL traffic is ignored and
-                the hostname should be used. The supplied value is interpreted as a
-                regular expression and matched on the ip or the hostname.
+            Ignore host and forward all traffic without processing it. In
+            transparent mode, it is recommended to use an IP address (range),
+            not the hostname. In regular mode, only SSL traffic is ignored and
+            the hostname should be used. The supplied value is interpreted as a
+            regular expression and matched on the ip or the hostname.
             """
         )
         self.add_option(
@@ -267,9 +286,9 @@ class Options(optmanager.OptManager):
         self.add_option(
             "mode", "regular", str,
             """
-                Mode can be "regular", "transparent", "socks5", "reverse:SPEC",
-                or "upstream:SPEC". For reverse and upstream proxy modes, SPEC
-                is proxy specification in the form of "http[s]://host[:port]".
+            Mode can be "regular", "transparent", "socks5", "reverse:SPEC",
+            or "upstream:SPEC". For reverse and upstream proxy modes, SPEC
+            is proxy specification in the form of "http[s]://host[:port]".
             """
         )
         self.add_option(
@@ -278,8 +297,10 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "keep_host_header", False, bool,
-            "Reverse Proxy: Keep the original host header instead of rewriting it"
-            " to the reverse proxy target."
+            """
+            Reverse Proxy: Keep the original host header instead of rewriting it
+            to the reverse proxy target.
+            """
         )
 
         self.add_option(
@@ -289,9 +310,11 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "http2_priority", False, bool,
-            "Enable/disable PRIORITY forwarding for HTTP/2 connections. "
-            "PRIORITY forwarding is disabled by default, "
-            "because some webservers fail to implement the RFC properly.",
+            """
+            PRIORITY forwarding for HTTP/2 connections. PRIORITY forwarding is
+            disabled by default, because some webservers fail to implement the
+            RFC properly.
+            """
         )
         self.add_option(
             "websocket", True, bool,
@@ -306,28 +329,32 @@ class Options(optmanager.OptManager):
 
         self.add_option(
             "spoof_source_address", False, bool,
-            "Use the client's IP for server-side connections. "
-            "Combine with --upstream-bind-address to spoof a fixed source address."
+            """
+            Use the client's IP for server-side connections. Combine with
+            --upstream-bind-address to spoof a fixed source address.
+            """
         )
         self.add_option(
             "upstream_auth", None, Optional[str],
             """
-                Add HTTP Basic authentcation to upstream proxy and reverse proxy
-                requests. Format: username:password
+            Add HTTP Basic authentcation to upstream proxy and reverse proxy
+            requests. Format: username:password
             """
         )
         self.add_option(
             "ssl_version_client", "secure", str,
-            "Set supported SSL/TLS versions for client connections. "
-            "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which "
-            "is TLS1.0+.",
+            """
+            Set supported SSL/TLS versions for client connections. SSLv2, SSLv3
+            and 'all' are INSECURE. Defaults to secure, which is TLS1.0+.
+            """,
             choices=tcp.sslversion_choices.keys(),
         )
         self.add_option(
             "ssl_version_server", "secure", str,
-            "Set supported SSL/TLS versions for server connections. "
-            "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, "
-            "which is TLS1.0+.",
+            """
+            Set supported SSL/TLS versions for server connections. SSLv2, SSLv3
+            and 'all' are INSECURE. Defaults to secure, which is TLS1.0+.
+            """,
             choices=tcp.sslversion_choices.keys(),
         )
         self.add_option(
@@ -336,8 +363,10 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "ssl_verify_upstream_trusted_cadir", None, Optional[str],
-            "Path to a directory of trusted CA certificates for upstream "
-            "server verification prepared using the c_rehash tool."
+            """
+            Path to a directory of trusted CA certificates for upstream server
+            verification prepared using the c_rehash tool.
+            """
         )
         self.add_option(
             "ssl_verify_upstream_trusted_ca", None, Optional[str],
@@ -346,9 +375,9 @@ class Options(optmanager.OptManager):
         self.add_option(
             "tcp_hosts", [], Sequence[str],
             """
-                Generic TCP SSL proxy mode for all hosts that match the pattern.
-                Similar to --ignore, but SSL connections are intercepted. The
-                communication contents are printed to the log in verbose mode.
+            Generic TCP SSL proxy mode for all hosts that match the pattern.
+            Similar to --ignore, but SSL connections are intercepted. The
+            communication contents are printed to the log in verbose mode.
             """
         )
 
@@ -368,7 +397,7 @@ class Options(optmanager.OptManager):
         )
         self.add_option(
             "console_palette", "dark", Optional[str],
-            help="Select color palette: " + ", ".join(console_palettes),
+            "Color palette.",
             choices=sorted(console_palettes),
         )
         self.add_option(
@@ -384,7 +413,10 @@ class Options(optmanager.OptManager):
             "Flow sort order.",
             choices=view_orders,
         )
-        self.add_option("console_order_reversed", False, bool)
+        self.add_option(
+            "console_order_reversed", False, bool,
+            "Reverse the sorting order."
+        )
 
         self.add_option(
             "filter", None, Optional[str],
@@ -410,7 +442,10 @@ class Options(optmanager.OptManager):
         )
 
         # Dump options
-        self.add_option("filtstr", None, Optional[str])
+        self.add_option(
+            "filtstr", None, Optional[str],
+            "The filter string for mitmdump"
+        )
         self.add_option(
             "flow_detail", 1, int,
             "Flow detail display level"
diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py
index e5277371..f03055eb 100644
--- a/mitmproxy/optmanager.py
+++ b/mitmproxy/optmanager.py
@@ -28,7 +28,7 @@ class _Option:
         name: str,
         default: typing.Any,
         typespec: typing.Type,
-        help: typing.Optional[str],
+        help: str,
         choices: typing.Optional[typing.Sequence[str]]
     ) -> None:
         typecheck.check_type(name, default, typespec)
@@ -103,7 +103,7 @@ class OptManager:
         name: str,
         default: typing.Any,
         typespec: typing.Type,
-        help: typing.Optional[str] = None,
+        help: str,
         choices: typing.Optional[typing.Sequence[str]] = None
     ) -> None:
         if name in self._options:
@@ -404,7 +404,7 @@ class OptManager:
                 action="append",
                 type=str,
                 dest=optname,
-                help=o.help,
+                help=o.help + " May be passed multiple times.",
                 metavar=metavar,
                 choices=o.choices,
             )
@@ -421,11 +421,10 @@ def dump(opts):
     for k in sorted(opts.keys()):
         o = opts._options[k]
         s[k] = o.default
-        if o.help:
-            s.yaml_set_comment_before_after_key(
-                k,
-                before = "\n" + "\n".join(textwrap.wrap(
-                    textwrap.dedent(o.help.strip())
-                )),
-            )
+        s.yaml_set_comment_before_after_key(
+            k,
+            before = "\n" + "\n".join(textwrap.wrap(
+                textwrap.dedent(o.help.strip())
+            )),
+        )
     return ruamel.yaml.round_trip_dump(s)
diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py
index d4f3aa69..bb33c1e4 100644
--- a/mitmproxy/tools/cmdline.py
+++ b/mitmproxy/tools/cmdline.py
@@ -91,26 +91,12 @@ def common_options(parser, opts):
     opts.make_parser(group, "server_replay_nopop")
 
     # Replacements
-    group = parser.add_argument_group(
-        "Replacements",
-        """
-            Replacements are of the form "/pattern/regex/replacement", where
-            the separator can be any character. Please see the documentation
-            for more information.
-        """.strip()
-    )
+    group = parser.add_argument_group("Replacements")
     opts.make_parser(group, "replacements", metavar="PATTERN")
     opts.make_parser(group, "replacement_files", metavar="PATTERN")
 
     # Set headers
-    group = parser.add_argument_group(
-        "Set Headers",
-        """
-            Header specifications are of the form "/pattern/header/value",
-            where the separator can be any character. Please see the
-            documentation for more information.
-        """.strip()
-    )
+    group = parser.add_argument_group("Set Headers")
     opts.make_parser(group, "setheaders", metavar="PATTERN")
 
     # Proxy authentication
@@ -149,8 +135,8 @@ def mitmdump(opts):
         'filter_args',
         nargs="...",
         help="""
-            Filter view expression, used to only show flows that match a certain filter.
-            See help in mitmproxy for filter expression syntax.
+            Filter view expression, used to only show flows that match a certain
+            filter. See help in mitmproxy for filter expression syntax.
         """
     )
     return parser
diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py
index 9311e82d..d6ce87e6 100644
--- a/test/mitmproxy/test_optmanager.py
+++ b/test/mitmproxy/test_optmanager.py
@@ -13,36 +13,36 @@ from mitmproxy.test import tutils
 class TO(optmanager.OptManager):
     def __init__(self):
         super().__init__()
-        self.add_option("one", None, typing.Optional[int])
-        self.add_option("two", 2, typing.Optional[int])
-        self.add_option("bool", False, bool)
+        self.add_option("one", None, typing.Optional[int], "help")
+        self.add_option("two", 2, typing.Optional[int], "help")
+        self.add_option("bool", False, bool, "help")
 
 
 class TD(optmanager.OptManager):
     def __init__(self):
         super().__init__()
-        self.add_option("one", "done", str)
-        self.add_option("two", "dtwo", str)
+        self.add_option("one", "done", str, "help")
+        self.add_option("two", "dtwo", str, "help")
 
 
 class TD2(TD):
     def __init__(self):
         super().__init__()
-        self.add_option("three", "dthree", str)
-        self.add_option("four", "dfour", str)
+        self.add_option("three", "dthree", str, "help")
+        self.add_option("four", "dfour", str, "help")
 
 
 class TM(optmanager.OptManager):
     def __init__(self):
         super().__init__()
-        self.add_option("two", ["foo"], typing.Sequence[str])
-        self.add_option("one", None, typing.Optional[str])
+        self.add_option("two", ["foo"], typing.Sequence[str], "help")
+        self.add_option("one", None, typing.Optional[str], "help")
 
 
 def test_add_option():
     o = TO()
     with pytest.raises(ValueError, match="already exists"):
-        o.add_option("one", None, typing.Optional[int])
+        o.add_option("one", None, typing.Optional[int], "help")
 
 
 def test_defaults():
@@ -291,13 +291,13 @@ def test_dump():
 class TTypes(optmanager.OptManager):
     def __init__(self):
         super().__init__()
-        self.add_option("str", "str", str)
-        self.add_option("optstr", "optstr", typing.Optional[str])
-        self.add_option("bool", False, bool)
-        self.add_option("int", 0, int)
-        self.add_option("optint", 0, typing.Optional[int])
-        self.add_option("seqstr", [], typing.Sequence[str])
-        self.add_option("unknown", 0.0, float)
+        self.add_option("str", "str", str, "help")
+        self.add_option("optstr", "optstr", typing.Optional[str], "help", "help")
+        self.add_option("bool", False, bool, "help")
+        self.add_option("int", 0, int, "help")
+        self.add_option("optint", 0, typing.Optional[int], "help")
+        self.add_option("seqstr", [], typing.Sequence[str], "help")
+        self.add_option("unknown", 0.0, float, "help")
 
 
 def test_make_parser():