First-order integration of configargparser to add config file support

6 files changed, 165 insertions, 71 deletions

diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index 4a3b5a48..09e25ada 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 import re
-from argparse import ArgumentTypeError
+from configargparse import ArgumentTypeError
 from netlib import http
 from . import filt, utils
 from .proxy import config
@@ -22,7 +22,9 @@ def _parse_hook(s):
     elif len(parts) == 3:
         patt, a, b = parts
     else:
-        raise ParseException("Malformed hook specifier - too few clauses: %s" % s)
+        raise ParseException(
+            "Malformed hook specifier - too few clauses: %s" % s
+        )
 
     if not a:
         raise ParseException("Empty clause: %s" % str(patt))
@@ -184,12 +186,16 @@ def common_options(parser):
     parser.add_argument(
         "--anticache",
         action="store_true", dest="anticache", default=False,
-        help="Strip out request headers that might cause the server to return 304-not-modified."
+
+        help="""
+            Strip out request headers that might cause the server to return
+            304-not-modified.
+        """
     )
     parser.add_argument(
-        "--confdir",
-        action="store", type=str, dest="confdir", default='~/.mitmproxy',
-        help="Configuration directory, contains default CA file. (~/.mitmproxy)"
+        "--cadir",
+        action="store", type=str, dest="cadir", default=config.CA_DIR,
+        help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
     )
     parser.add_argument(
         "--host",
@@ -210,11 +216,17 @@ def common_options(parser):
         "-s",
         action="append", type=str, dest="scripts", default=[],
         metavar='"script.py --bar"',
-        help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
+        help="""
+            Run a script. Surround with quotes to pass script arguments. Can be
+            passed multiple times.
+        """
     )
     parser.add_argument(
         "-t",
-        action="store", dest="stickycookie_filt", default=None, metavar="FILTER",
+        action="store",
+        dest="stickycookie_filt",
+        default=None,
+        metavar="FILTER",
         help="Set sticky cookie filter. Matched against requests."
     )
     parser.add_argument(
@@ -241,7 +253,7 @@ def common_options(parser):
         "-Z",
         action="store", dest="body_size_limit", default=None,
         metavar="SIZE",
-        help="Byte size limit of HTTP request and response bodies." \
+        help="Byte size limit of HTTP request and response bodies."
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
     parser.add_argument(
@@ -249,9 +261,9 @@ def common_options(parser):
         action="store", dest="stream_large_bodies", default=None,
         metavar="SIZE",
         help="""
-        Stream data to the client if response body exceeds the given threshold.
-        If streamed, the body will not be stored in any way. Understands k/m/g
-        suffixes, i.e. 3m for 3 megabytes.
+            Stream data to the client if response body exceeds the given
+            threshold. If streamed, the body will not be stored in any way.
+            Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
          """
     )
 
@@ -282,8 +294,11 @@ def common_options(parser):
         "--tcp",
         action="append", type=str, dest="tcp_hosts", default=[],
         metavar="HOST",
-        help="Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore,"
-             "but SSL connections are intercepted. The communication contents are printed to the event log in verbose mode."
+        help="""
+            Generic TCP SSL proxy mode for all hosts that match the pattern.
+            Similar to --ignore, but SSL connections are intercepted. The
+            communication contents are printed to the event log in verbose mode.
+        """
     )
     group.add_argument(
         "-n",
@@ -297,8 +312,14 @@ def common_options(parser):
     )
     group.add_argument(
         "-R",
-        action="store", type=parse_server_spec, dest="reverse_proxy", default=None,
-        help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]"
+        action="store",
+        type=parse_server_spec,
+        dest="reverse_proxy",
+        default=None,
+        help="""
+            Forward all requests to upstream HTTP server:
+            http[s][2http[s]]://host[:port]
+        """
     )
     group.add_argument(
         "--socks",
@@ -312,16 +333,20 @@ def common_options(parser):
     )
     group.add_argument(
         "-U",
-        action="store", type=parse_server_spec, dest="upstream_proxy", default=None,
+        action="store",
+        type=parse_server_spec,
+        dest="upstream_proxy",
+        default=None,
         help="Forward all requests to upstream proxy server: http://host[:port]"
     )
 
     group = parser.add_argument_group(
         "Advanced Proxy Options",
         """
-            The following options allow a custom adjustment of the proxy behavior.
-            Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -U, -T).
-        """.strip()
+            The following options allow a custom adjustment of the proxy
+            behavior. Normally, you don't want to use these options directly and
+            use the provided wrappers instead (-R, -U, -T).
+        """
     )
     group.add_argument(
         "--http-form-in", dest="http_form_in", default=None,
@@ -343,13 +368,19 @@ def common_options(parser):
     group.add_argument(
         "--app-host",
         action="store", dest="app_host", default=APP_HOST, metavar="host",
-        help="Domain to serve the onboarding app from. For transparent mode, use an IP when\
-                a DNS entry for the app domain is not present. Default: %s" % APP_HOST
-
+        help="""
+            Domain to serve the onboarding app from. For transparent mode, use
+            an IP when a DNS entry for the app domain is not present. Default:
+            %s
+        """ % APP_HOST
     )
     group.add_argument(
         "--app-port",
-        action="store", dest="app_port", default=APP_PORT, type=int, metavar="80",
+        action="store",
+        dest="app_port",
+        default=APP_PORT,
+        type=int,
+        metavar="80",
         help="Port to serve the onboarding app from."
     )
 
@@ -380,8 +411,10 @@ def common_options(parser):
     group.add_argument(
         "--norefresh",
         action="store_true", dest="norefresh", default=False,
-        help="Disable response refresh, "
-             "which updates times in cookies and headers for replayed responses."
+        help="""
+            Disable response refresh, which updates times in cookies and headers
+            for replayed responses.
+        """
     )
     group.add_argument(
         "--no-pop",
@@ -392,13 +425,17 @@ def common_options(parser):
     group.add_argument(
         "--replay-ignore-content",
         action="store_true", dest="replay_ignore_content", default=False,
-        help="Ignore request's content while searching for a saved flow to replay"
+        help="""
+            Ignore request's content while searching for a saved flow to replay
+        """
     )
     group.add_argument(
         "--replay-ignore-param",
         action="append", dest="replay_ignore_params", type=str,
-        help="Request's parameters to be ignored while searching for a saved flow to replay"
-           "Can be passed multiple times."
+        help="""
+            Request's parameters to be ignored while searching for a saved flow
+            to replay. Can be passed multiple times.
+        """
     )
 
     group = parser.add_argument_group(
@@ -417,9 +454,12 @@ def common_options(parser):
     )
     group.add_argument(
         "--replace-from-file",
-        action="append", type=str, dest="replace_file", default=[],
-        metavar="PATH",
-        help="Replacement pattern, where the replacement clause is a path to a file."
+        action = "append", type=str, dest="replace_file", default=[],
+        metavar = "PATH",
+        help = """
+            Replacement pattern, where the replacement clause is a path to a
+            file.
+        """
     )
 
     group = parser.add_argument_group(
@@ -455,7 +495,10 @@ def common_options(parser):
         "--singleuser",
         action="store", dest="auth_singleuser", type=str,
         metavar="USER",
-        help="Allows access to a a single user, specified in the form username:password."
+        help="""
+            Allows access to a a single user, specified in the form
+            username:password.
+        """
     )
     user_specification_group.add_argument(
         "--htpasswd",
diff --git a/libmproxy/main.py b/libmproxy/main.py
index 2d6a0119..9cad5dcc 100644
--- a/libmproxy/main.py
+++ b/libmproxy/main.py
@@ -1,4 +1,5 @@
 from __future__ import print_function, absolute_import
+import configargparse
 import argparse
 import os
 import signal
@@ -11,25 +12,38 @@ from .proxy.server import DummyServer, ProxyServer
 
 def check_versions():
     """
-    Having installed a wrong version of pyOpenSSL or netlib is unfortunately a very common source of error.
-    Check before every start that both versions are somewhat okay.
+    Having installed a wrong version of pyOpenSSL or netlib is unfortunately a
+    very common source of error. Check before every start that both versions are
+    somewhat okay.
     """
-    # We don't introduce backward-incompatible changes in patch versions. Only consider major and minor version.
+    # We don't introduce backward-incompatible changes in patch versions. Only
+    # consider major and minor version.
     if netlib.version.IVERSION[:2] != version.IVERSION[:2]:
         print(
             "Warning: You are using mitmdump %s with netlib %s. "
-            "Most likely, that doesn't work - please upgrade!" % (version.VERSION, netlib.version.VERSION),
-            file=sys.stderr)
-    import OpenSSL, inspect
-
+            "Most likely, that won't work - please upgrade!" % (
+                version.VERSION, netlib.version.VERSION
+            ),
+            file=sys.stderr
+        )
+    import OpenSSL
+    import inspect
     v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2])
     if v < (0, 14):
-        print("You are using an outdated version of pyOpenSSL: mitmproxy requires pyOpenSSL 0.14 or greater.",
-              file=sys.stderr)
-        # Some users apparently have multiple versions of pyOpenSSL installed. Report which one we got.
+        print(
+            "You are using an outdated version of pyOpenSSL:"
+            " mitmproxy requires pyOpenSSL 0.14 or greater.",
+            file=sys.stderr
+        )
+        # Some users apparently have multiple versions of pyOpenSSL installed.
+        # Report which one we got.
         pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL))
-        print("Your pyOpenSSL %s installation is located at %s" % (OpenSSL.__version__, pyopenssl_path),
-              file=sys.stderr)
+        print(
+            "Your pyOpenSSL %s installation is located at %s" % (
+                OpenSSL.__version__, pyopenssl_path
+            ),
+            file=sys.stderr
+        )
         sys.exit(1)
 
 
@@ -38,8 +52,14 @@ def assert_utf8_env():
     for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
         spec += os.environ.get(i, "").lower()
     if "utf" not in spec:
-        print("Error: mitmproxy requires a UTF console environment.", file=sys.stderr)
-        print("Set your LANG enviroment variable to something like en_US.UTF-8", file=sys.stderr)
+        print(
+            "Error: mitmproxy requires a UTF console environment.",
+            file=sys.stderr
+        )
+        print(
+            "Set your LANG enviroment variable to something like en_US.UTF-8",
+            file=sys.stderr
+        )
         sys.exit(1)
 
 
@@ -55,12 +75,17 @@ def get_server(dummy_server, options):
 
 
 def mitmproxy_cmdline():
-    # Don't import libmproxy.console for mitmdump, urwid is not available on all platforms.
+    # Don't import libmproxy.console for mitmdump, urwid is not available on all
+    # platforms.
     from . import console
     from .console import palettes
 
-    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
-    parser.add_argument('--version', action='version', version=version.NAMEVERSION)
+    parser = configargparse.ArgumentParser(usage="%(prog)s [options]")
+    parser.add_argument(
+        '--version',
+        action='version',
+        version=version.NAMEVERSION
+    )
     cmdline.common_options(parser)
     parser.add_argument(
         "--palette", type=str, default="dark",
@@ -113,13 +138,21 @@ def mitmproxy():  # pragma: nocover
 def mitmdump_cmdline():
     from . import dump
 
-    parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]")
-    parser.add_argument('--version', action='version', version="mitmdump" + " " + version.VERSION)
+    parser = configargparse.ArgumentParser(usage="%(prog)s [options] [filter]")
+
+    parser.add_argument(
+        '--version',
+        action= 'version',
+        version= "mitmdump" + " " + version.VERSION
+    )
     cmdline.common_options(parser)
     parser.add_argument(
         "--keepserving",
-        action="store_true", dest="keepserving", default=False,
-        help="Continue serving after client playback or file read. We exit by default."
+        action= "store_true", dest="keepserving", default=False,
+        help= """
+            Continue serving after client playback or file read. We exit by
+            default.
+        """
     )
     parser.add_argument(
         "-d",
@@ -166,7 +199,7 @@ def mitmdump():  # pragma: nocover
 
 def mitmweb_cmdline():
     from . import web
-    parser = argparse.ArgumentParser(usage="%(prog)s [options]")
+    parser = configargparse.ArgumentParser(usage="%(prog)s [options]")
     parser.add_argument(
         '--version',
         action='version',
diff --git a/libmproxy/onboarding/app.py b/libmproxy/onboarding/app.py
index 9b5db38a..4023fae2 100644
--- a/libmproxy/onboarding/app.py
+++ b/libmproxy/onboarding/app.py
@@ -18,12 +18,12 @@ def index():
 
 @mapp.route("/cert/pem")
 def certs_pem():
-    p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem")
+    p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem")
     return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert')
 
 
 @mapp.route("/cert/p12")
 def certs_p12():
-    p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12")
+    p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12")
     return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12')
 
diff --git a/libmproxy/platform/windows.py b/libmproxy/platform/windows.py
index ddbbed52..066a377d 100644
--- a/libmproxy/platform/windows.py
+++ b/libmproxy/platform/windows.py
@@ -1,4 +1,4 @@
-import argparse
+import configargparse
 import cPickle as pickle
 from ctypes import byref, windll, Structure
 from ctypes.wintypes import DWORD
@@ -361,7 +361,7 @@ class TransparentProxy(object):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Windows Transparent Proxy")
+    parser = configargparse.ArgumentParser(description="Windows Transparent Proxy")
     parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both",
                         help='redirection operation mode: "forward" to only redirect forwarded packets, '
                              '"local" to only redirect packets originating from the local machine')
diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py
index a228192a..3d373a28 100644
--- a/libmproxy/proxy/config.py
+++ b/libmproxy/proxy/config.py
@@ -7,7 +7,7 @@ from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMod
 
 TRANSPARENT_SSL_PORTS = [443, 8443]
 CONF_BASENAME = "mitmproxy"
-CONF_DIR = "~/.mitmproxy"
+CA_DIR = "~/.mitmproxy"
 
 
 class HostMatcher(object):
@@ -28,12 +28,27 @@ class HostMatcher(object):
 
 
 class ProxyConfig:
-    def __init__(self, host='', port=8080, server_version=version.NAMEVERSION,
-                 confdir=CONF_DIR, clientcerts=None,
-                 no_upstream_cert=False, body_size_limit=None,
-                 mode=None, upstream_server=None, http_form_in=None, http_form_out=None,
-                 authenticator=None, ignore_hosts=[], tcp_hosts=[],
-                 ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS):
+    def __init__(
+            self,
+            host='',
+            port=8080,
+            server_version=version.NAMEVERSION,
+            cadir=CA_DIR,
+            clientcerts=None,
+            no_upstream_cert=False,
+            body_size_limit=None,
+            mode=None,
+            upstream_server=None,
+            http_form_in=None,
+            http_form_out=None,
+            authenticator=None,
+            ignore_hosts=[],
+            tcp_hosts=[],
+            ciphers=None,
+            certs=[],
+            certforward=False,
+            ssl_ports=TRANSPARENT_SSL_PORTS
+    ):
         self.host = host
         self.port = port
         self.server_version = server_version
@@ -60,8 +75,8 @@ class ProxyConfig:
         self.check_ignore = HostMatcher(ignore_hosts)
         self.check_tcp = HostMatcher(tcp_hosts)
         self.authenticator = authenticator
-        self.confdir = os.path.expanduser(confdir)
-        self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME)
+        self.cadir = os.path.expanduser(cadir)
+        self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME)
         for spec, cert in certs:
             self.certstore.add_cert_file(spec, cert)
         self.certforward = certforward
@@ -136,7 +151,7 @@ def process_proxy_options(parser, options):
     return ProxyConfig(
         host=options.addr,
         port=options.port,
-        confdir=options.confdir,
+        cadir=options.cadir,
         clientcerts=options.clientcerts,
         no_upstream_cert=options.no_upstream_cert,
         body_size_limit=body_size_limit,
diff --git a/setup.py b/setup.py
index 08ccbbfd..55c2a87e 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,8 @@ deps = {
     "pyOpenSSL>=0.14",
     "Flask>=0.10.1",
     "tornado>=4.0.2",
-    "sortedcontainers>=0.9.1"
+    "sortedcontainers>=0.9.1",
+    "configargparse>=0.9.3"
 }
 script_deps = {
     "mitmproxy": {
@@ -72,7 +73,9 @@ setup(
             "nose>=1.3.0",
             "nose-cov>=1.6",
             "coveralls>=0.4.1",
-            "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)
+            "pathod>=%s, <%s" % (
+                version.MINORVERSION, version.NEXT_MINORVERSION
+            )
         ],
         'contentviews': [
             "pyamf>=0.6.1",