diff options
author | Aldo Cortesi <aldo@corte.si> | 2016-10-20 12:36:26 +1300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-10-20 12:36:26 +1300 |
commit | 711078ba3f63257df745bb3edd80a85717e94b20 (patch) | |
tree | d3116cd540faf01f272a0892fc6a9b83b4f6de8a | |
parent | ee56d3fae0baeef1f31a83db122dd832d4c0e07e (diff) | |
parent | 8430f857b504a3e7406dc36e54dc32783569d0dd (diff) | |
download | mitmproxy-711078ba3f63257df745bb3edd80a85717e94b20.tar.gz mitmproxy-711078ba3f63257df745bb3edd80a85717e94b20.tar.bz2 mitmproxy-711078ba3f63257df745bb3edd80a85717e94b20.zip |
Merge pull request #1637 from cortesi/tatanetlib
This PR merges netlib into mitmproxy
-rw-r--r-- | .appveyor.yml | 2 | ||||
-rw-r--r-- | .travis.yml | 2 | ||||
-rw-r--r-- | MANIFEST.in | 3 | ||||
-rw-r--r-- | README.rst | 7 | ||||
-rw-r--r-- | docs/conf.py | 11 | ||||
-rw-r--r-- | docs/dev/testing.rst | 2 | ||||
-rw-r--r-- | docs/modd.conf | 2 | ||||
-rw-r--r-- | examples/custom_contentviews.py | 2 | ||||
-rw-r--r-- | examples/har_dump.py | 6 | ||||
-rw-r--r-- | examples/tcp_message.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/dumper.py | 4 | ||||
-rw-r--r-- | mitmproxy/addons/onboardingapp/app.py | 6 | ||||
-rw-r--r-- | mitmproxy/addons/serverplayback.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/stickycookie.py | 2 | ||||
-rw-r--r-- | mitmproxy/addons/streambodies.py | 4 | ||||
-rw-r--r-- | mitmproxy/addons/termlog.py | 4 | ||||
-rw-r--r-- | mitmproxy/addons/wsgiapp.py | 4 | ||||
-rw-r--r-- | mitmproxy/certs.py (renamed from netlib/certutils.py) | 6 | ||||
-rw-r--r-- | mitmproxy/connections.py | 8 | ||||
-rw-r--r-- | mitmproxy/contentviews.py | 8 | ||||
-rw-r--r-- | mitmproxy/exceptions.py | 86 | ||||
-rw-r--r-- | mitmproxy/export.py | 6 | ||||
-rw-r--r-- | mitmproxy/flow.py | 2 | ||||
-rw-r--r-- | mitmproxy/flowfilter.py | 2 | ||||
-rw-r--r-- | mitmproxy/http.py | 14 | ||||
-rw-r--r-- | mitmproxy/io_compat.py | 3 | ||||
-rw-r--r-- | mitmproxy/log.py | 4 | ||||
-rw-r--r-- | mitmproxy/master.py | 6 | ||||
-rw-r--r-- | mitmproxy/net/__init__.py (renamed from netlib/__init__.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/check.py | 22 | ||||
-rw-r--r-- | mitmproxy/net/http/__init__.py | 15 | ||||
-rw-r--r-- | mitmproxy/net/http/authentication.py (renamed from netlib/http/authentication.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/http/cookies.py (renamed from netlib/http/cookies.py) | 2 | ||||
-rw-r--r-- | mitmproxy/net/http/encoding.py (renamed from netlib/encoding.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/http/headers.py (renamed from netlib/http/headers.py) | 4 | ||||
-rw-r--r-- | mitmproxy/net/http/http1/__init__.py (renamed from netlib/http/http1/__init__.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/http/http1/assemble.py (renamed from netlib/http/http1/assemble.py) | 10 | ||||
-rw-r--r-- | mitmproxy/net/http/http1/read.py (renamed from netlib/http/http1/read.py) | 14 | ||||
-rw-r--r-- | mitmproxy/net/http/http2/__init__.py | 8 | ||||
-rw-r--r-- | mitmproxy/net/http/http2/framereader.py (renamed from netlib/http/http2/framereader.py) | 4 | ||||
-rw-r--r-- | mitmproxy/net/http/http2/utils.py (renamed from netlib/http/http2/utils.py) | 2 | ||||
-rw-r--r-- | mitmproxy/net/http/message.py (renamed from netlib/http/message.py) | 12 | ||||
-rw-r--r-- | mitmproxy/net/http/multipart.py (renamed from netlib/http/multipart.py) | 2 | ||||
-rw-r--r-- | mitmproxy/net/http/request.py (renamed from netlib/http/request.py) | 40 | ||||
-rw-r--r-- | mitmproxy/net/http/response.py (renamed from netlib/http/response.py) | 14 | ||||
-rw-r--r-- | mitmproxy/net/http/status_codes.py (renamed from netlib/http/status_codes.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/http/url.py (renamed from netlib/http/url.py) | 6 | ||||
-rw-r--r-- | mitmproxy/net/http/user_agents.py (renamed from netlib/http/user_agents.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/socks.py (renamed from netlib/socks.py) | 18 | ||||
-rw-r--r-- | mitmproxy/net/tcp.py (renamed from netlib/tcp.py) | 24 | ||||
-rw-r--r-- | mitmproxy/net/websockets/__init__.py (renamed from netlib/websockets/__init__.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/websockets/frame.py (renamed from netlib/websockets/frame.py) | 33 | ||||
-rw-r--r-- | mitmproxy/net/websockets/masker.py (renamed from netlib/websockets/masker.py) | 0 | ||||
-rw-r--r-- | mitmproxy/net/websockets/utils.py (renamed from netlib/websockets/utils.py) | 3 | ||||
-rw-r--r-- | mitmproxy/net/wsgi.py (renamed from netlib/wsgi.py) | 4 | ||||
-rw-r--r-- | mitmproxy/proxy/config.py | 12 | ||||
-rw-r--r-- | mitmproxy/proxy/modes/socks_proxy.py | 5 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/base.py | 3 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/http.py | 29 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/http1.py | 2 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/http2.py | 15 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/http_replay.py | 7 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/rawtcp.py | 8 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/tls.py | 11 | ||||
-rw-r--r-- | mitmproxy/proxy/protocol/websockets.py | 9 | ||||
-rw-r--r-- | mitmproxy/proxy/root_context.py | 3 | ||||
-rw-r--r-- | mitmproxy/proxy/server.py | 7 | ||||
-rw-r--r-- | mitmproxy/script/concurrent.py | 2 | ||||
-rw-r--r-- | mitmproxy/stateobject.py | 4 | ||||
-rw-r--r-- | mitmproxy/tcp.py | 4 | ||||
-rw-r--r-- | mitmproxy/test/tutils.py (renamed from netlib/tutils.py) | 12 | ||||
-rw-r--r-- | mitmproxy/tools/cmdline.py | 6 | ||||
-rw-r--r-- | mitmproxy/tools/console/common.py | 12 | ||||
-rw-r--r-- | mitmproxy/tools/console/flowdetailview.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/console/flowlist.py | 4 | ||||
-rw-r--r-- | mitmproxy/tools/console/flowview.py | 8 | ||||
-rw-r--r-- | mitmproxy/tools/console/grideditor/col_bytes.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/console/grideditor/col_subgrid.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/console/grideditor/editors.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/console/help.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/console/master.py | 8 | ||||
-rw-r--r-- | mitmproxy/tools/console/statusbar.py | 6 | ||||
-rw-r--r-- | mitmproxy/tools/dump.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/main.py | 4 | ||||
-rw-r--r-- | mitmproxy/tools/web/app.py | 2 | ||||
-rw-r--r-- | mitmproxy/tools/web/master.py | 2 | ||||
-rw-r--r-- | mitmproxy/types/__init__.py (renamed from test/netlib/__init__.py) | 0 | ||||
-rw-r--r-- | mitmproxy/types/basethread.py (renamed from netlib/basethread.py) | 0 | ||||
-rw-r--r-- | mitmproxy/types/bidi.py | 29 | ||||
-rw-r--r-- | mitmproxy/types/multidict.py (renamed from netlib/multidict.py) | 4 | ||||
-rw-r--r-- | mitmproxy/types/serializable.py (renamed from netlib/basetypes.py) | 0 | ||||
-rw-r--r-- | mitmproxy/utils/__init__.py (renamed from test/netlib/http/__init__.py) | 0 | ||||
-rw-r--r-- | mitmproxy/utils/bits.py | 15 | ||||
-rw-r--r-- | mitmproxy/utils/data.py | 33 | ||||
-rw-r--r-- | mitmproxy/utils/debug.py (renamed from netlib/debug.py) | 4 | ||||
-rw-r--r-- | mitmproxy/utils/human.py (renamed from netlib/human.py) | 0 | ||||
-rw-r--r-- | mitmproxy/utils/lrucache.py (renamed from mitmproxy/utils.py) | 8 | ||||
-rw-r--r-- | mitmproxy/utils/strutils.py (renamed from netlib/strutils.py) | 0 | ||||
-rw-r--r-- | mitmproxy/utils/version_check.py (renamed from netlib/version_check.py) | 5 | ||||
-rw-r--r-- | mitmproxy/version.py (renamed from netlib/version.py) | 0 | ||||
-rw-r--r-- | netlib/exceptions.py | 59 | ||||
-rw-r--r-- | netlib/http/__init__.py | 15 | ||||
-rw-r--r-- | netlib/http/http2/__init__.py | 8 | ||||
-rw-r--r-- | netlib/utils.py | 97 | ||||
-rw-r--r-- | pathod/language/base.py | 4 | ||||
-rw-r--r-- | pathod/language/http.py | 12 | ||||
-rw-r--r-- | pathod/language/http2.py | 4 | ||||
-rw-r--r-- | pathod/language/message.py | 2 | ||||
-rw-r--r-- | pathod/language/websockets.py | 20 | ||||
-rw-r--r-- | pathod/language/writer.py | 4 | ||||
-rw-r--r-- | pathod/log.py | 3 | ||||
-rw-r--r-- | pathod/pathoc.py | 22 | ||||
-rw-r--r-- | pathod/pathoc_cmdline.py | 6 | ||||
-rw-r--r-- | pathod/pathod.py | 28 | ||||
-rw-r--r-- | pathod/pathod_cmdline.py | 6 | ||||
-rw-r--r-- | pathod/protocols/http.py | 8 | ||||
-rw-r--r-- | pathod/protocols/http2.py | 26 | ||||
-rw-r--r-- | pathod/protocols/websockets.py | 8 | ||||
-rw-r--r-- | pathod/test.py | 2 | ||||
-rw-r--r-- | pathod/utils.py | 5 | ||||
-rw-r--r-- | release/README.mkd | 2 | ||||
-rwxr-xr-x | release/rtool.py | 2 | ||||
-rw-r--r-- | setup.py | 3 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_dumper.py | 10 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_serverplayback.py | 6 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_stickycookie.py | 2 | ||||
-rw-r--r-- | test/mitmproxy/console/test_master.py | 12 | ||||
-rw-r--r-- | test/mitmproxy/mastertest.py | 6 | ||||
-rw-r--r-- | test/mitmproxy/net/__init__.py (renamed from test/netlib/http/http1/__init__.py) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/clientcert/.gitignore (renamed from test/netlib/data/clientcert/.gitignore) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/clientcert/client.cnf (renamed from test/netlib/data/clientcert/client.cnf) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/clientcert/client.pem (renamed from test/netlib/data/clientcert/client.pem) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/clientcert/make (renamed from test/netlib/data/clientcert/make) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/dercert (renamed from test/netlib/data/dercert) | bin | 1838 -> 1838 bytes | |||
-rw-r--r-- | test/mitmproxy/net/data/dhparam.pem (renamed from test/netlib/data/dhparam.pem) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/htpasswd (renamed from test/netlib/data/htpasswd) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/server.crt (renamed from test/netlib/data/server.crt) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/server.key (renamed from test/netlib/data/server.key) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/text_cert (renamed from test/netlib/data/text_cert) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/text_cert_2 (renamed from test/netlib/data/text_cert_2) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/text_cert_weird1 (renamed from test/netlib/data/text_cert_weird1) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/9da13359.0 (renamed from test/netlib/data/verificationcerts/9da13359.0) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/generate.py (renamed from test/netlib/data/verificationcerts/generate.py) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/self-signed.crt (renamed from test/netlib/data/verificationcerts/self-signed.crt) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/self-signed.key (renamed from test/netlib/data/verificationcerts/self-signed.key) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/trusted-leaf.crt (renamed from test/netlib/data/verificationcerts/trusted-leaf.crt) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/trusted-leaf.key (renamed from test/netlib/data/verificationcerts/trusted-leaf.key) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/trusted-root.crt (renamed from test/netlib/data/verificationcerts/trusted-root.crt) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/trusted-root.key (renamed from test/netlib/data/verificationcerts/trusted-root.key) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/data/verificationcerts/trusted-root.srl (renamed from test/netlib/data/verificationcerts/trusted-root.srl) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/http/__init__.py (renamed from test/netlib/http/http2/__init__.py) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/http/http1/__init__.py (renamed from test/netlib/websockets/__init__.py) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/http/http1/test_assemble.py (renamed from test/netlib/http/http1/test_assemble.py) | 12 | ||||
-rw-r--r-- | test/mitmproxy/net/http/http1/test_read.py (renamed from test/netlib/http/http1/test_read.py) | 61 | ||||
-rw-r--r-- | test/mitmproxy/net/http/http2/__init__.py | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/http/http2/test_framereader.py (renamed from test/netlib/http/http2/test_framereader.py) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_authentication.py (renamed from test/netlib/http/test_authentication.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_cookies.py (renamed from test/netlib/http/test_cookies.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_encoding.py (renamed from test/netlib/test_encoding.py) | 3 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_headers.py (renamed from test/netlib/http/test_headers.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_message.py (renamed from test/netlib/http/test_message.py) | 58 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_multipart.py (renamed from test/netlib/http/test_multipart.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_request.py (renamed from test/netlib/http/test_request.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_response.py (renamed from test/netlib/http/test_response.py) | 8 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_status_codes.py (renamed from test/netlib/http/test_status_codes.py) | 2 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_url.py (renamed from test/netlib/http/test_url.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/http/test_user_agents.py (renamed from test/netlib/http/test_user_agents.py) | 2 | ||||
-rw-r--r-- | test/mitmproxy/net/test_check.py | 10 | ||||
-rw-r--r-- | test/mitmproxy/net/test_imports.py (renamed from test/netlib/test_imports.py) | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/test_socks.py (renamed from test/netlib/test_socks.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/test_tcp.py (renamed from test/netlib/test_tcp.py) | 53 | ||||
-rw-r--r-- | test/mitmproxy/net/test_wsgi.py (renamed from test/netlib/test_wsgi.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/tools/getcertnames (renamed from test/netlib/tools/getcertnames) | 2 | ||||
-rw-r--r-- | test/mitmproxy/net/tservers.py (renamed from test/netlib/tservers.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/websockets/__init__.py | 0 | ||||
-rw-r--r-- | test/mitmproxy/net/websockets/test_frame.py (renamed from test/netlib/websockets/test_frame.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/net/websockets/test_masker.py (renamed from test/netlib/websockets/test_masker.py) | 2 | ||||
-rw-r--r-- | test/mitmproxy/net/websockets/test_utils.py (renamed from test/netlib/websockets/test_utils.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/protocol/test_http1.py | 6 | ||||
-rw-r--r-- | test/mitmproxy/protocol/test_http2.py | 44 | ||||
-rw-r--r-- | test/mitmproxy/protocol/test_websockets.py | 25 | ||||
-rw-r--r-- | test/mitmproxy/test_certs.py (renamed from test/netlib/test_certutils.py) | 41 | ||||
-rw-r--r-- | test/mitmproxy/test_contentview.py | 10 | ||||
-rw-r--r-- | test/mitmproxy/test_controller.py | 2 | ||||
-rw-r--r-- | test/mitmproxy/test_custom_contentview.py | 2 | ||||
-rw-r--r-- | test/mitmproxy/test_examples.py | 11 | ||||
-rw-r--r-- | test/mitmproxy/test_flow.py | 30 | ||||
-rw-r--r-- | test/mitmproxy/test_flow_export.py | 10 | ||||
-rw-r--r-- | test/mitmproxy/test_optmanager.py | 2 | ||||
-rw-r--r-- | test/mitmproxy/test_proxy.py | 6 | ||||
-rw-r--r-- | test/mitmproxy/test_server.py | 51 | ||||
-rw-r--r-- | test/mitmproxy/test_types_bidi.py | 11 | ||||
-rw-r--r-- | test/mitmproxy/test_types_multidict.py (renamed from test/netlib/test_multidict.py) | 10 | ||||
-rw-r--r-- | test/mitmproxy/test_types_serializable.py (renamed from test/netlib/test_basetypes.py) | 4 | ||||
-rw-r--r-- | test/mitmproxy/test_utils_data.py | 7 | ||||
-rw-r--r-- | test/mitmproxy/test_utils_debug.py (renamed from test/netlib/test_debug.py) | 2 | ||||
-rw-r--r-- | test/mitmproxy/test_utils_human.py (renamed from test/netlib/test_human.py) | 3 | ||||
-rw-r--r-- | test/mitmproxy/test_utils_lrucache.py (renamed from test/mitmproxy/test_utils.py) | 12 | ||||
-rw-r--r-- | test/mitmproxy/test_utils_strutils.py (renamed from test/netlib/test_strutils.py) | 3 | ||||
-rw-r--r-- | test/mitmproxy/test_utils_version_check.py (renamed from test/netlib/test_version_check.py) | 2 | ||||
-rw-r--r-- | test/mitmproxy/tservers.py | 7 | ||||
-rw-r--r-- | test/mitmproxy/tutils.py | 12 | ||||
-rw-r--r-- | test/netlib/test_utils.py | 19 | ||||
-rw-r--r-- | test/pathod/test_language_http2.py | 4 | ||||
-rw-r--r-- | test/pathod/test_language_websocket.py | 12 | ||||
-rw-r--r-- | test/pathod/test_log.py | 4 | ||||
-rw-r--r-- | test/pathod/test_pathoc.py | 12 | ||||
-rw-r--r-- | test/pathod/test_pathod.py | 12 | ||||
-rw-r--r-- | test/pathod/test_protocols_http2.py | 32 | ||||
-rw-r--r-- | test/pathod/tutils.py | 9 | ||||
-rw-r--r-- | tox.ini | 2 |
211 files changed, 907 insertions, 893 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 38868628..746e0fd1 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -23,7 +23,7 @@ install: - "pip install -U tox" test_script: - - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v" + - ps: "tox -- --cov mitmproxy --cov pathod -v" deploy_script: ps: | diff --git a/.travis.yml b/.travis.yml index d160a96e..d7cb7027 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ install: fi - pip install tox -script: tox -- --cov netlib --cov mitmproxy --cov pathod -v +script: tox -- --cov mitmproxy --cov pathod -v after_success: - | diff --git a/MANIFEST.in b/MANIFEST.in index 440a14ed..404936e8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,3 @@ graft mitmproxy graft pathod -graft netlib -recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
\ No newline at end of file +recursive-exclude * *.pyc *.pyo *.swo *.swp *.map @@ -3,8 +3,7 @@ mitmproxy |travis| |appveyor| |coverage| |latest_release| |python_versions| -This repository contains the **mitmproxy** and **pathod** projects, as well as -their shared networking library, **netlib**. +This repository contains the **mitmproxy** and **pathod** projects. ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface. @@ -64,7 +63,7 @@ virtualenv_ installed (you can find installation instructions for virtualenv The *dev* script will create a virtualenv environment in a directory called "venv", and install all mandatory and optional dependencies into it. The -primary mitmproxy components - mitmproxy, netlib and pathod - are installed as +primary mitmproxy components - mitmproxy and pathod - are installed as "editable", so any changes to the source in the repository will be reflected live in the virtualenv. @@ -144,7 +143,7 @@ PR checks will fail and block merging. We are using this command to check for st .. code-block:: text - flake8 --jobs 8 --count mitmproxy netlib pathod examples test + flake8 --jobs 8 --count mitmproxy pathod examples test .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg diff --git a/docs/conf.py b/docs/conf.py index e1cbc497..d7793b79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,7 +5,7 @@ import subprocess import sys sys.path.insert(0, os.path.abspath('..')) -import netlib.version +from mitmproxy import version extensions = [ @@ -47,9 +47,9 @@ author = u'The mitmproxy project' # built documents. # # The short X.Y version. -version = netlib.version.VERSION +version = version.VERSION # The full version, including alpha/beta/rc tags. -release = netlib.version.VERSION +release = version.VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -231,10 +231,7 @@ def linkcode_resolve(domain, info): _, line = inspect.getsourcelines(obj) except (TypeError, IOError): return None - if spath.rfind("netlib") > -1: - off = spath.rfind("netlib") - mpath = spath[off:] - elif spath.rfind("mitmproxy") > -1: + if spath.rfind("mitmproxy") > -1: off = spath.rfind("mitmproxy") mpath = spath[off:] else: diff --git a/docs/dev/testing.rst b/docs/dev/testing.rst index 315b9bc9..b86efce1 100644 --- a/docs/dev/testing.rst +++ b/docs/dev/testing.rst @@ -10,7 +10,7 @@ suitable extension to the test suite. Our tests are written for the `py.test`_ or nose_ test frameworks. At the point where you send your pull request, a command like this: ->>> py.test --cov mitmproxy --cov netlib +>>> py.test --cov mitmproxy Should give output something like this: diff --git a/docs/modd.conf b/docs/modd.conf index bc91238c..0e8f0dce 100644 --- a/docs/modd.conf +++ b/docs/modd.conf @@ -1,6 +1,6 @@ @build = ./_build -** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py { +** !_build/** ../mitmproxy/**/*.py { prep: sphinx-build -W -d @build/doctrees -b html . @build/html daemon: devd -m @build/html } diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py index 5a63e2a0..3558eaca 100644 --- a/examples/custom_contentviews.py +++ b/examples/custom_contentviews.py @@ -2,7 +2,7 @@ import string import lxml.html import lxml.etree from mitmproxy import contentviews -from netlib import strutils +from mitmproxy.utils import strutils class ViewPigLatin(contentviews.View): diff --git a/examples/har_dump.py b/examples/har_dump.py index 95090edb..560b9adc 100644 --- a/examples/har_dump.py +++ b/examples/har_dump.py @@ -14,9 +14,9 @@ import pytz import mitmproxy -from netlib import version -from netlib import strutils -from netlib.http import cookies +from mitmproxy import version +from mitmproxy.utils import strutils +from mitmproxy.net.http import cookies HAR = {} diff --git a/examples/tcp_message.py b/examples/tcp_message.py index b431c23f..d7c9c42e 100644 --- a/examples/tcp_message.py +++ b/examples/tcp_message.py @@ -8,7 +8,7 @@ tcp_message Inline Script Hook API Demonstration example cmdline invocation: mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py """ -from netlib import strutils +from mitmproxy.utils import strutils def tcp_message(tcp_msg): diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 04dfb42c..d690c000 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -8,8 +8,8 @@ from mitmproxy import contentviews from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import flowfilter -from netlib import human -from netlib import strutils +from mitmproxy.utils import human +from mitmproxy.utils import strutils def indent(n, text): diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py index 9e07b75f..50b52214 100644 --- a/mitmproxy/addons/onboardingapp/app.py +++ b/mitmproxy/addons/onboardingapp/app.py @@ -4,11 +4,11 @@ import tornado.template import tornado.web import tornado.wsgi -from mitmproxy import utils +from mitmproxy.utils import data from mitmproxy.proxy import config from mitmproxy.addons import wsgiapp -loader = tornado.template.Loader(utils.pkg_data.path("addons/onboardingapp/templates")) +loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates")) class Adapter(tornado.wsgi.WSGIAdapter): @@ -86,7 +86,7 @@ application = tornado.web.Application( r"/static/(.*)", tornado.web.StaticFileHandler, { - "path": utils.pkg_data.path("addons/onboardingapp/static") + "path": data.pkg_data.path("addons/onboardingapp/static") } ), ], diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 1161ce23..0b52918c 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,7 +1,7 @@ import urllib import hashlib -from netlib import strutils +from mitmproxy.utils import strutils from mitmproxy import exceptions from mitmproxy import ctx from mitmproxy import io diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index d89bd92d..27d78646 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -1,7 +1,7 @@ import collections from http import cookiejar -from netlib.http import cookies +from mitmproxy.net.http import cookies from mitmproxy import exceptions from mitmproxy import flowfilter diff --git a/mitmproxy/addons/streambodies.py b/mitmproxy/addons/streambodies.py index b3e5d6b2..88cb74f3 100644 --- a/mitmproxy/addons/streambodies.py +++ b/mitmproxy/addons/streambodies.py @@ -1,5 +1,5 @@ -from netlib.http import http1 -from netlib import exceptions +from mitmproxy.net.http import http1 +from mitmproxy import exceptions from mitmproxy import ctx diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 50c32044..05be32d0 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -1,6 +1,6 @@ import click -from mitmproxy import utils +from mitmproxy import log class TermLog: @@ -11,7 +11,7 @@ class TermLog: self.options = options def log(self, e): - if self.options.verbosity >= utils.log_tier(e.level): + if self.options.verbosity >= log.log_tier(e.level): click.secho( e.msg, file=self.options.tfile, diff --git a/mitmproxy/addons/wsgiapp.py b/mitmproxy/addons/wsgiapp.py index d83a1e2e..85d09a0a 100644 --- a/mitmproxy/addons/wsgiapp.py +++ b/mitmproxy/addons/wsgiapp.py @@ -1,8 +1,8 @@ from mitmproxy import ctx from mitmproxy import exceptions -from netlib import wsgi -from netlib import version +from mitmproxy.net import wsgi +from mitmproxy import version class WSGIApp: diff --git a/netlib/certutils.py b/mitmproxy/certs.py index 6a97f99e..83db4e6e 100644 --- a/netlib/certutils.py +++ b/mitmproxy/certs.py @@ -10,7 +10,7 @@ from pyasn1.codec.der.decoder import decode from pyasn1.error import PyAsn1Error import OpenSSL -from netlib import basetypes +from mitmproxy.types import serializable # Default expiry must not be too long: https://github.com/mitmproxy/mitmproxy/issues/815 @@ -193,7 +193,7 @@ class CertStore: @staticmethod def load_dhparam(path): - # netlib<=0.10 doesn't generate a dhparam file. + # mitmproxy<=0.10 doesn't generate a dhparam file. # Create it now if neccessary. if not os.path.exists(path): with open(path, "wb") as f: @@ -373,7 +373,7 @@ class _GeneralNames(univ.SequenceOf): constraint.ValueSizeConstraint(1, 1024) -class SSLCert(basetypes.Serializable): +class SSLCert(serializable.Serializable): def __init__(self, cert): """ diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index bf7a12aa..015d0689 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -4,8 +4,8 @@ import copy import os from mitmproxy import stateobject -from netlib import certutils -from netlib import tcp +from mitmproxy import certs +from mitmproxy.net import tcp class ClientConnection(tcp.BaseHandler, stateobject.StateObject): @@ -57,7 +57,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): _stateobject_attributes = dict( address=tcp.Address, ssl_established=bool, - clientcert=certutils.SSLCert, + clientcert=certs.SSLCert, timestamp_start=float, timestamp_ssl_setup=float, timestamp_end=float, @@ -151,7 +151,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ip_address=tcp.Address, source_address=tcp.Address, ssl_established=bool, - cert=certutils.SSLCert, + cert=certs.SSLCert, sni=str, timestamp_start=float, timestamp_tcp_setup=float, diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 9a70b104..b7f15d8d 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -33,10 +33,10 @@ from PIL import ExifTags from PIL import Image from mitmproxy import exceptions from mitmproxy.contrib.wbxml import ASCommandResponse -from netlib import http -from netlib import multidict -from netlib import strutils -from netlib.http import url +from mitmproxy.net import http +from mitmproxy.types import multidict +from mitmproxy.utils import strutils +from mitmproxy.net.http import url try: import pyamf diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index 64cc457a..309b8189 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -1,13 +1,16 @@ """ We try to be very hygienic regarding the exceptions we throw: -Every Exception mitmproxy raises shall be a subclass of ProxyException. +- Every exception that might be externally visible to users shall be a subclass + of MitmproxyException.p +- Every exception in the base net module shall be a subclass + of NetlibException, and will not be propagated directly to users. See also: http://lucumr.pocoo.org/2014/10/16/on-error-handling/ """ -class ProxyException(Exception): +class MitmproxyException(Exception): """ Base class for all exceptions thrown by mitmproxy. @@ -17,7 +20,7 @@ class ProxyException(Exception): super().__init__(message) -class Kill(ProxyException): +class Kill(MitmproxyException): """ Signal that both client and server connection(s) should be killed immediately. @@ -25,7 +28,7 @@ class Kill(ProxyException): pass -class ProtocolException(ProxyException): +class ProtocolException(MitmproxyException): """ ProtocolExceptions are caused by invalid user input, unavailable network resources, or other events that are outside of our influence. @@ -66,37 +69,94 @@ class Http2ZombieException(ProtocolException): pass -class ServerException(ProxyException): +class ServerException(MitmproxyException): pass -class ContentViewException(ProxyException): +class ContentViewException(MitmproxyException): pass -class ReplayException(ProxyException): +class ReplayException(MitmproxyException): pass -class FlowReadException(ProxyException): +class FlowReadException(MitmproxyException): pass -class ControlException(ProxyException): +class ControlException(MitmproxyException): pass -class SetServerNotAllowedException(ProxyException): +class SetServerNotAllowedException(MitmproxyException): pass -class OptionsError(Exception): +class OptionsError(MitmproxyException): pass -class AddonError(Exception): +class AddonError(MitmproxyException): pass -class AddonHalt(Exception): +class AddonHalt(MitmproxyException): + pass + + +""" + Net-layer exceptions +""" + + +class NetlibException(MitmproxyException): + """ + Base class for all exceptions thrown by mitmproxy.net. + """ + def __init__(self, message=None): + super().__init__(message) + + +class Disconnect: + """Immediate EOF""" + + +class HttpException(NetlibException): + pass + + +class HttpReadDisconnect(HttpException, Disconnect): + pass + + +class HttpSyntaxException(HttpException): + pass + + +class TcpException(NetlibException): + pass + + +class TcpDisconnect(TcpException, Disconnect): + pass + + +class TcpReadIncomplete(TcpException): + pass + + +class TcpTimeout(TcpException): + pass + + +class TlsException(NetlibException): + pass + + +class InvalidCertificateException(TlsException): + pass + + +class Timeout(TcpException): pass diff --git a/mitmproxy/export.py b/mitmproxy/export.py index e5f4d34a..d9a88849 100644 --- a/mitmproxy/export.py +++ b/mitmproxy/export.py @@ -3,7 +3,7 @@ import re import textwrap import urllib -import netlib.http +import mitmproxy.net.http def _native(s): @@ -89,9 +89,9 @@ def python_code(flow): return code -def is_json(headers: netlib.http.Headers, content: bytes) -> bool: +def is_json(headers: mitmproxy.net.http.Headers, content: bytes) -> bool: if headers: - ct = netlib.http.parse_content_type(headers.get("content-type", "")) + ct = mitmproxy.net.http.parse_content_type(headers.get("content-type", "")) if ct and "%s/%s" % (ct[0], ct[1]) == "application/json": try: return json.loads(content.decode("utf8", "surrogateescape")) diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 13b852ef..2fb7ace0 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -4,8 +4,8 @@ import uuid from mitmproxy import stateobject from mitmproxy import connections +from mitmproxy import version -from netlib import version from typing import Optional # noqa diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index f1454fd1..ee5224c6 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -40,7 +40,7 @@ from mitmproxy import http from mitmproxy import tcp from mitmproxy import flow -from netlib import strutils +from mitmproxy.utils import strutils import pyparsing as pp from typing import Callable diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 7fe70f9b..99e126fe 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1,9 +1,9 @@ import cgi from mitmproxy import flow -from netlib import http -from netlib import version -from netlib import tcp +from mitmproxy.net import http +from mitmproxy import version +from mitmproxy.net import tcp class HTTPRequest(http.Request): @@ -12,7 +12,7 @@ class HTTPRequest(http.Request): A mitmproxy HTTP request. """ - # This is a very thin wrapper on top of :py:class:`netlib.http.Request` and + # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Request` and # may be removed in the future. def __init__( @@ -73,7 +73,7 @@ class HTTPRequest(http.Request): @classmethod def wrap(self, request): """ - Wraps an existing :py:class:`netlib.http.Request`. + Wraps an existing :py:class:`mitmproxy.net.http.Request`. """ req = HTTPRequest( first_line_format=request.data.first_line_format, @@ -99,7 +99,7 @@ class HTTPResponse(http.Response): """ A mitmproxy HTTP response. """ - # This is a very thin wrapper on top of :py:class:`netlib.http.Response` and + # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and # may be removed in the future. def __init__( @@ -131,7 +131,7 @@ class HTTPResponse(http.Response): @classmethod def wrap(self, response): """ - Wraps an existing :py:class:`netlib.http.Response`. + Wraps an existing :py:class:`mitmproxy.net.http.Response`. """ resp = HTTPResponse( http_version=response.data.http_version, diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py index fc190e2f..68c747ea 100644 --- a/mitmproxy/io_compat.py +++ b/mitmproxy/io_compat.py @@ -4,7 +4,8 @@ This module handles the import of mitmproxy flows generated by old versions. from typing import Any -from netlib import version, strutils +from mitmproxy import version +from mitmproxy.utils import strutils def convert_011_012(data): diff --git a/mitmproxy/log.py b/mitmproxy/log.py index 8c28a9b1..c2456cf1 100644 --- a/mitmproxy/log.py +++ b/mitmproxy/log.py @@ -38,3 +38,7 @@ class Log: def __call__(self, text, level="info"): self.master.add_log(text, level) + + +def log_tier(level): + return dict(error=0, warn=1, info=2, debug=3).get(level) diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 1fc00112..31ce17a3 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -14,8 +14,8 @@ from mitmproxy import http from mitmproxy import log from mitmproxy import io from mitmproxy.proxy.protocol import http_replay -from netlib import basethread -import netlib.http +from mitmproxy.types import basethread +import mitmproxy.net.http from . import ctx as mitmproxy_ctx @@ -122,7 +122,7 @@ class Master: s = connections.ServerConnection.make_dummy((host, port)) f = http.HTTPFlow(c, s) - headers = netlib.http.Headers() + headers = mitmproxy.net.http.Headers() req = http.HTTPRequest( "absolute", diff --git a/netlib/__init__.py b/mitmproxy/net/__init__.py index e69de29b..e69de29b 100644 --- a/netlib/__init__.py +++ b/mitmproxy/net/__init__.py diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py new file mode 100644 index 00000000..7b007cb5 --- /dev/null +++ b/mitmproxy/net/check.py @@ -0,0 +1,22 @@ +import re + +_label_valid = re.compile(b"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE) + + +def is_valid_host(host: bytes) -> bool: + """ + Checks if a hostname is valid. + """ + try: + host.decode("idna") + except ValueError: + return False + if len(host) > 255: + return False + if host and host[-1:] == b".": + host = host[:-1] + return all(_label_valid.match(x) for x in host.split(b".")) + + +def is_valid_port(port): + return 0 <= port <= 65535 diff --git a/mitmproxy/net/http/__init__.py b/mitmproxy/net/http/__init__.py new file mode 100644 index 00000000..7e290bdc --- /dev/null +++ b/mitmproxy/net/http/__init__.py @@ -0,0 +1,15 @@ +from mitmproxy.net.http.request import Request +from mitmproxy.net.http.response import Response +from mitmproxy.net.http.message import Message +from mitmproxy.net.http.headers import Headers, parse_content_type +from mitmproxy.net.http.message import decoded +from mitmproxy.net.http import http1, http2, status_codes, multipart + +__all__ = [ + "Request", + "Response", + "Message", + "Headers", "parse_content_type", + "decoded", + "http1", "http2", "status_codes", "multipart", +] diff --git a/netlib/http/authentication.py b/mitmproxy/net/http/authentication.py index a65279e4..a65279e4 100644 --- a/netlib/http/authentication.py +++ b/mitmproxy/net/http/authentication.py diff --git a/netlib/http/cookies.py b/mitmproxy/net/http/cookies.py index cb816ca0..9f32fa5e 100644 --- a/netlib/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -3,7 +3,7 @@ import email.utils import re import time -from netlib import multidict +from mitmproxy.types import multidict """ A flexible module for cookie parsing and manipulation. diff --git a/netlib/encoding.py b/mitmproxy/net/http/encoding.py index e123a033..e123a033 100644 --- a/netlib/encoding.py +++ b/mitmproxy/net/http/encoding.py diff --git a/netlib/http/headers.py b/mitmproxy/net/http/headers.py index 39673f1a..8fc0cd43 100644 --- a/netlib/http/headers.py +++ b/mitmproxy/net/http/headers.py @@ -1,8 +1,8 @@ import re import collections -from netlib import multidict -from netlib import strutils +from mitmproxy.types import multidict +from mitmproxy.utils import strutils # See also: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/ diff --git a/netlib/http/http1/__init__.py b/mitmproxy/net/http/http1/__init__.py index e4bf01c5..e4bf01c5 100644 --- a/netlib/http/http1/__init__.py +++ b/mitmproxy/net/http/http1/__init__.py diff --git a/netlib/http/http1/assemble.py b/mitmproxy/net/http/http1/assemble.py index 3d65da34..d718589f 100644 --- a/netlib/http/http1/assemble.py +++ b/mitmproxy/net/http/http1/assemble.py @@ -1,5 +1,5 @@ -import netlib.http.url -from netlib import exceptions +import mitmproxy.net.http.url +from mitmproxy import exceptions def assemble_request(request): @@ -44,7 +44,7 @@ def assemble_body(headers, body_chunks): def _assemble_request_line(request_data): """ Args: - request_data (netlib.http.request.RequestData) + request_data (mitmproxy.net.http.request.RequestData) """ form = request_data.first_line_format if form == "relative": @@ -76,11 +76,11 @@ def _assemble_request_line(request_data): def _assemble_request_headers(request_data): """ Args: - request_data (netlib.http.request.RequestData) + request_data (mitmproxy.net.http.request.RequestData) """ headers = request_data.headers.copy() if "host" not in headers and request_data.scheme and request_data.host and request_data.port: - headers["host"] = netlib.http.url.hostport( + headers["host"] = mitmproxy.net.http.url.hostport( request_data.scheme, request_data.host, request_data.port diff --git a/netlib/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 4c00a96a..6eb30709 100644 --- a/netlib/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -2,12 +2,12 @@ import time import sys import re -from netlib.http import request -from netlib.http import response -from netlib.http import headers -from netlib.http import url -from netlib import utils -from netlib import exceptions +from mitmproxy.net.http import request +from mitmproxy.net.http import response +from mitmproxy.net.http import headers +from mitmproxy.net.http import url +from mitmproxy.net import check +from mitmproxy import exceptions def get_header_tokens(headers, key): @@ -274,7 +274,7 @@ def _parse_authority_form(hostport): try: host, port = hostport.split(b":") port = int(port) - if not utils.is_valid_host(host) or not utils.is_valid_port(port): + if not check.is_valid_host(host) or not check.is_valid_port(port): raise ValueError() except ValueError: raise exceptions.HttpSyntaxException("Invalid host specification: {}".format(hostport)) diff --git a/mitmproxy/net/http/http2/__init__.py b/mitmproxy/net/http/http2/__init__.py new file mode 100644 index 00000000..7027006b --- /dev/null +++ b/mitmproxy/net/http/http2/__init__.py @@ -0,0 +1,8 @@ +from mitmproxy.net.http.http2.framereader import read_raw_frame, parse_frame +from mitmproxy.net.http.http2.utils import parse_headers + +__all__ = [ + "read_raw_frame", + "parse_frame", + "parse_headers", +] diff --git a/netlib/http/http2/framereader.py b/mitmproxy/net/http/http2/framereader.py index 8b7cfffb..6a164919 100644 --- a/netlib/http/http2/framereader.py +++ b/mitmproxy/net/http/http2/framereader.py @@ -1,7 +1,7 @@ import codecs import hyperframe -from ...exceptions import HttpException +from mitmproxy import exceptions def read_raw_frame(rfile): @@ -9,7 +9,7 @@ def read_raw_frame(rfile): length = int(codecs.encode(header[:3], 'hex_codec'), 16) if length == 4740180: - raise HttpException("Length field looks more like HTTP/1.1:\n{}".format(rfile.read(-1))) + raise exceptions.HttpException("Length field looks more like HTTP/1.1:\n{}".format(rfile.read(-1))) body = rfile.safe_read(length) return [header, body] diff --git a/netlib/http/http2/utils.py b/mitmproxy/net/http/http2/utils.py index 164bacc8..62a59c72 100644 --- a/netlib/http/http2/utils.py +++ b/mitmproxy/net/http/http2/utils.py @@ -1,4 +1,4 @@ -from netlib.http import url +from mitmproxy.net.http import url def parse_headers(headers): diff --git a/netlib/http/message.py b/mitmproxy/net/http/message.py index 1980b0ab..af1d16be 100644 --- a/netlib/http/message.py +++ b/mitmproxy/net/http/message.py @@ -2,8 +2,10 @@ import re import warnings from typing import Optional -from netlib import encoding, strutils, basetypes -from netlib.http import headers +from mitmproxy.utils import strutils +from mitmproxy.net.http import encoding +from mitmproxy.types import serializable +from mitmproxy.net.http import headers # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded. @@ -15,7 +17,7 @@ def _always_bytes(x): return strutils.always_bytes(x, "utf-8", "surrogateescape") -class MessageData(basetypes.Serializable): +class MessageData(serializable.Serializable): def __eq__(self, other): if isinstance(other, MessageData): return self.__dict__ == other.__dict__ @@ -41,7 +43,7 @@ class MessageData(basetypes.Serializable): return cls(**state) -class Message(basetypes.Serializable): +class Message(serializable.Serializable): def __eq__(self, other): if isinstance(other, Message): return self.data == other.data @@ -67,7 +69,7 @@ class Message(basetypes.Serializable): Message headers object Returns: - netlib.http.Headers + mitmproxy.net.http.Headers """ return self.data.headers diff --git a/netlib/http/multipart.py b/mitmproxy/net/http/multipart.py index 536b2809..a854d47f 100644 --- a/netlib/http/multipart.py +++ b/mitmproxy/net/http/multipart.py @@ -1,6 +1,6 @@ import re -from netlib.http import headers +from mitmproxy.net.http import headers def decode(hdrs, content): diff --git a/netlib/http/request.py b/mitmproxy/net/http/request.py index dd6f4164..c3d85363 100644 --- a/netlib/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,13 +1,13 @@ import re import urllib -from netlib import multidict -from netlib import strutils -from netlib.http import multipart -from netlib.http import cookies -from netlib.http import headers as nheaders -from netlib.http import message -import netlib.http.url +from mitmproxy.types import multidict +from mitmproxy.utils import strutils +from mitmproxy.net.http import multipart +from mitmproxy.net.http import cookies +from mitmproxy.net.http import headers as nheaders +from mitmproxy.net.http import message +import mitmproxy.net.http.url # This regex extracts & splits the host header into host and port. # Handles the edge case of IPv6 addresses containing colons. @@ -203,11 +203,11 @@ class Request(message.Message): """ if self.first_line_format == "authority": return "%s:%d" % (self.host, self.port) - return netlib.http.url.unparse(self.scheme, self.host, self.port, self.path) + return mitmproxy.net.http.url.unparse(self.scheme, self.host, self.port, self.path) @url.setter def url(self, url): - self.scheme, self.host, self.port, self.path = netlib.http.url.parse(url) + self.scheme, self.host, self.port, self.path = mitmproxy.net.http.url.parse(url) def _parse_host_header(self): """Extract the host and port from Host header""" @@ -243,12 +243,12 @@ class Request(message.Message): """ if self.first_line_format == "authority": return "%s:%d" % (self.pretty_host, self.port) - return netlib.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path) + return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path) @property def query(self) -> multidict.MultiDictView: """ - The request query string as an :py:class:`~netlib.multidict.MultiDictView` object. + The request query string as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. """ return multidict.MultiDictView( self._get_query, @@ -257,10 +257,10 @@ class Request(message.Message): def _get_query(self): query = urllib.parse.urlparse(self.url).query - return tuple(netlib.http.url.decode(query)) + return tuple(mitmproxy.net.http.url.decode(query)) def _set_query(self, query_data): - query = netlib.http.url.encode(query_data) + query = mitmproxy.net.http.url.encode(query_data) _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) @@ -273,7 +273,7 @@ class Request(message.Message): """ The request cookies. - An empty :py:class:`~netlib.multidict.MultiDictView` object if the cookie monster ate them all. + An empty :py:class:`~mitmproxy.net.multidict.MultiDictView` object if the cookie monster ate them all. """ return multidict.MultiDictView( self._get_cookies, @@ -301,11 +301,11 @@ class Request(message.Message): # This needs to be a tuple so that it's immutable. # Otherwise, this would fail silently: # request.path_components.append("foo") - return tuple(netlib.http.url.unquote(i) for i in path.split("/") if i) + return tuple(mitmproxy.net.http.url.unquote(i) for i in path.split("/") if i) @path_components.setter def path_components(self, components): - components = map(lambda x: netlib.http.url.quote(x, safe=""), components) + components = map(lambda x: mitmproxy.net.http.url.quote(x, safe=""), components) path = "/" + "/".join(components) _, _, _, params, query, fragment = urllib.parse.urlparse(self.url) self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) @@ -347,7 +347,7 @@ class Request(message.Message): @property def urlencoded_form(self): """ - The URL-encoded form data as an :py:class:`~netlib.multidict.MultiDictView` object. + The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. An empty multidict.MultiDictView if the content-type indicates non-form data or the content could not be parsed. """ @@ -360,7 +360,7 @@ class Request(message.Message): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: try: - return tuple(netlib.http.url.decode(self.content)) + return tuple(mitmproxy.net.http.url.decode(self.content)) except ValueError: pass return () @@ -371,7 +371,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = netlib.http.url.encode(form_data).encode() + self.content = mitmproxy.net.http.url.encode(form_data).encode() @urlencoded_form.setter def urlencoded_form(self, value): @@ -380,7 +380,7 @@ class Request(message.Message): @property def multipart_form(self): """ - The multipart form data as an :py:class:`~netlib.multidict.MultiDictView` object. + The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. None if the content-type indicates non-form data. """ return multidict.MultiDictView( diff --git a/netlib/http/response.py b/mitmproxy/net/http/response.py index a8b48be0..b458a3d8 100644 --- a/netlib/http/response.py +++ b/mitmproxy/net/http/response.py @@ -1,11 +1,11 @@ import time from email.utils import parsedate_tz, formatdate, mktime_tz -from netlib import human -from netlib import multidict -from netlib.http import cookies -from netlib.http import headers as nheaders -from netlib.http import message -from netlib.http import status_codes +from mitmproxy.utils import human +from mitmproxy.types import multidict +from mitmproxy.net.http import cookies +from mitmproxy.net.http import headers as nheaders +from mitmproxy.net.http import message +from mitmproxy.net.http import status_codes from typing import AnyStr from typing import Dict from typing import Iterable @@ -131,7 +131,7 @@ class Response(message.Message): def cookies(self) -> multidict.MultiDictView: """ The response cookies. A possibly empty - :py:class:`~netlib.multidict.MultiDictView`, where the keys are cookie + :py:class:`~mitmproxy.net.multidict.MultiDictView`, where the keys are cookie name strings, and values are (value, attr) tuples. Value is a string, and attr is an MultiDictView containing cookie attributes. Within attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value. diff --git a/netlib/http/status_codes.py b/mitmproxy/net/http/status_codes.py index 5a83cd73..5a83cd73 100644 --- a/netlib/http/status_codes.py +++ b/mitmproxy/net/http/status_codes.py diff --git a/netlib/http/url.py b/mitmproxy/net/http/url.py index 67e22efa..ff3d5264 100644 --- a/netlib/http/url.py +++ b/mitmproxy/net/http/url.py @@ -2,7 +2,7 @@ import urllib from typing import Sequence from typing import Tuple -from netlib import utils +from mitmproxy.net import check # PY2 workaround @@ -62,9 +62,9 @@ def parse(url): if not full_path.startswith(b"/"): full_path = b"/" + full_path - if not utils.is_valid_host(host): + if not check.is_valid_host(host): raise ValueError("Invalid Host") - if not utils.is_valid_port(port): + if not check.is_valid_port(port): raise ValueError("Invalid Port") return parsed.scheme, host, port, full_path diff --git a/netlib/http/user_agents.py b/mitmproxy/net/http/user_agents.py index d0ca2f21..d0ca2f21 100644 --- a/netlib/http/user_agents.py +++ b/mitmproxy/net/http/user_agents.py diff --git a/netlib/socks.py b/mitmproxy/net/socks.py index 9f1adb98..115949f5 100644 --- a/netlib/socks.py +++ b/mitmproxy/net/socks.py @@ -2,7 +2,9 @@ import struct import array import ipaddress -from netlib import tcp, utils +from mitmproxy.net import tcp +from mitmproxy.net import check +from mitmproxy.types import bidi class SocksError(Exception): @@ -10,24 +12,24 @@ class SocksError(Exception): super().__init__(message) self.code = code -VERSION = utils.BiDi( +VERSION = bidi.BiDi( SOCKS4=0x04, SOCKS5=0x05 ) -CMD = utils.BiDi( +CMD = bidi.BiDi( CONNECT=0x01, BIND=0x02, UDP_ASSOCIATE=0x03 ) -ATYP = utils.BiDi( +ATYP = bidi.BiDi( IPV4_ADDRESS=0x01, DOMAINNAME=0x03, IPV6_ADDRESS=0x04 ) -REP = utils.BiDi( +REP = bidi.BiDi( SUCCEEDED=0x00, GENERAL_SOCKS_SERVER_FAILURE=0x01, CONNECTION_NOT_ALLOWED_BY_RULESET=0x02, @@ -39,14 +41,14 @@ REP = utils.BiDi( ADDRESS_TYPE_NOT_SUPPORTED=0x08, ) -METHOD = utils.BiDi( +METHOD = bidi.BiDi( NO_AUTHENTICATION_REQUIRED=0x00, GSSAPI=0x01, USERNAME_PASSWORD=0x02, NO_ACCEPTABLE_METHODS=0xFF ) -USERNAME_PASSWORD_VERSION = utils.BiDi( +USERNAME_PASSWORD_VERSION = bidi.BiDi( DEFAULT=0x01 ) @@ -203,7 +205,7 @@ class Message: elif atyp == ATYP.DOMAINNAME: length, = struct.unpack("!B", f.safe_read(1)) host = f.safe_read(length) - if not utils.is_valid_host(host): + if not check.is_valid_host(host): raise SocksError(REP.GENERAL_SOCKS_SERVER_FAILURE, "Invalid hostname: %s" % host) host = host.decode("idna") use_ipv6 = False diff --git a/netlib/tcp.py b/mitmproxy/net/tcp.py index aeb1d447..ac368a9c 100644 --- a/netlib/tcp.py +++ b/mitmproxy/net/tcp.py @@ -10,18 +10,18 @@ import binascii from typing import Optional # noqa -from netlib import strutils +from mitmproxy.utils import strutils import certifi from backports import ssl_match_hostname import OpenSSL from OpenSSL import SSL -from netlib import certutils -from netlib import version_check -from netlib import basetypes -from netlib import exceptions -from netlib import basethread +from mitmproxy import certs +from mitmproxy.utils import version_check +from mitmproxy.types import serializable +from mitmproxy import exceptions +from mitmproxy.types import basethread # This is a rather hackish way to make sure that # the latest version of pyOpenSSL is actually installed. @@ -292,7 +292,7 @@ class Reader(_FileLike): raise NotImplementedError("Can only peek into (pyOpenSSL) sockets") -class Address(basetypes.Serializable): +class Address(serializable.Serializable): """ This class wraps an IPv4/IPv6 tuple to provide named attributes and @@ -685,11 +685,11 @@ class TCPClient(_Connection): if verification_mode == SSL.VERIFY_PEER and self.ssl_verification_error: raise self.ssl_verification_error - self.cert = certutils.SSLCert(self.connection.get_peer_certificate()) + self.cert = certs.SSLCert(self.connection.get_peer_certificate()) # Keep all server certificates in a list for i in self.connection.get_peer_cert_chain(): - self.server_certs.append(certutils.SSLCert(i)) + self.server_certs.append(certs.SSLCert(i)) # Validate TLS Hostname try: @@ -782,7 +782,7 @@ class BaseHandler(_Connection): extra_chain_certs=None, **sslctx_kwargs): """ - cert: A certutils.SSLCert object or the path to a certificate + cert: A certs.SSLCert object or the path to a certificate chain file. handle_sni: SNI handler, should take a connection object. Server @@ -810,7 +810,7 @@ class BaseHandler(_Connection): context = self._create_ssl_context(ca_pemfile=chain_file, **sslctx_kwargs) context.use_privatekey(key) - if isinstance(cert, certutils.SSLCert): + if isinstance(cert, certs.SSLCert): context.use_certificate(cert.x509) else: context.use_certificate_chain_file(cert) @@ -825,7 +825,7 @@ class BaseHandler(_Connection): if request_client_cert: def save_cert(conn_, cert, errno_, depth_, preverify_ok_): - self.clientcert = certutils.SSLCert(cert) + self.clientcert = certs.SSLCert(cert) # Return true to prevent cert verification error return True context.set_verify(SSL.VERIFY_PEER, save_cert) diff --git a/netlib/websockets/__init__.py b/mitmproxy/net/websockets/__init__.py index 2d6f0a0c..2d6f0a0c 100644 --- a/netlib/websockets/__init__.py +++ b/mitmproxy/net/websockets/__init__.py diff --git a/netlib/websockets/frame.py b/mitmproxy/net/websockets/frame.py index b58fa289..bd5f67dd 100644 --- a/netlib/websockets/frame.py +++ b/mitmproxy/net/websockets/frame.py @@ -2,10 +2,11 @@ import os import struct import io -from netlib import tcp -from netlib import strutils -from netlib import utils -from netlib import human +from mitmproxy.net import tcp +from mitmproxy.utils import strutils +from mitmproxy.utils import bits +from mitmproxy.utils import human +from mitmproxy.types import bidi from .masker import Masker @@ -15,7 +16,7 @@ MAX_64_BIT_INT = (1 << 64) DEFAULT = object() # RFC 6455, Section 5.2 - Base Framing Protocol -OPCODE = utils.BiDi( +OPCODE = bidi.BiDi( CONTINUE=0x00, TEXT=0x01, BINARY=0x02, @@ -25,7 +26,7 @@ OPCODE = utils.BiDi( ) # RFC 6455, Section 7.4.1 - Defined Status Codes -CLOSE_REASON = utils.BiDi( +CLOSE_REASON = bidi.BiDi( NORMAL_CLOSURE=1000, GOING_AWAY=1001, PROTOCOL_ERROR=1002, @@ -118,13 +119,13 @@ class FrameHeader: return "".join(vals) def __bytes__(self): - first_byte = utils.setbit(0, 7, self.fin) - first_byte = utils.setbit(first_byte, 6, self.rsv1) - first_byte = utils.setbit(first_byte, 5, self.rsv2) - first_byte = utils.setbit(first_byte, 4, self.rsv3) + first_byte = bits.setbit(0, 7, self.fin) + first_byte = bits.setbit(first_byte, 6, self.rsv1) + first_byte = bits.setbit(first_byte, 5, self.rsv2) + first_byte = bits.setbit(first_byte, 4, self.rsv3) first_byte = first_byte | self.opcode - second_byte = utils.setbit(self.length_code, 7, self.mask) + second_byte = bits.setbit(self.length_code, 7, self.mask) b = bytes([first_byte, second_byte]) @@ -151,12 +152,12 @@ class FrameHeader: read a websockets frame header """ first_byte, second_byte = fp.safe_read(2) - fin = utils.getbit(first_byte, 7) - rsv1 = utils.getbit(first_byte, 6) - rsv2 = utils.getbit(first_byte, 5) - rsv3 = utils.getbit(first_byte, 4) + fin = bits.getbit(first_byte, 7) + rsv1 = bits.getbit(first_byte, 6) + rsv2 = bits.getbit(first_byte, 5) + rsv3 = bits.getbit(first_byte, 4) opcode = first_byte & 0xF - mask_bit = utils.getbit(second_byte, 7) + mask_bit = bits.getbit(second_byte, 7) length_code = second_byte & 0x7F # payload_length > 125 indicates you need to read more bytes diff --git a/netlib/websockets/masker.py b/mitmproxy/net/websockets/masker.py index 47b1a688..47b1a688 100644 --- a/netlib/websockets/masker.py +++ b/mitmproxy/net/websockets/masker.py diff --git a/netlib/websockets/utils.py b/mitmproxy/net/websockets/utils.py index fdec074e..d0b168ce 100644 --- a/netlib/websockets/utils.py +++ b/mitmproxy/net/websockets/utils.py @@ -8,7 +8,8 @@ import base64 import hashlib import os -from netlib import http, strutils +from mitmproxy.net import http +from mitmproxy.utils import strutils MAGIC = b'258EAFA5-E914-47DA-95CA-C5AB0DC85B11' VERSION = "13" diff --git a/netlib/wsgi.py b/mitmproxy/net/wsgi.py index 11e4aba9..b2705ea1 100644 --- a/netlib/wsgi.py +++ b/mitmproxy/net/wsgi.py @@ -3,7 +3,9 @@ import traceback import urllib import io -from netlib import http, tcp, strutils +from mitmproxy.net import http +from mitmproxy.net import tcp +from mitmproxy.utils import strutils class ClientConn: diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 201dfdf7..7df9e692 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -4,16 +4,16 @@ import os import re from typing import Any -from netlib import strutils +from mitmproxy.utils import strutils from OpenSSL import SSL, crypto from mitmproxy import exceptions from mitmproxy import options as moptions -from netlib import certutils -from netlib import tcp -from netlib.http import authentication -from netlib.http import url +from mitmproxy import certs +from mitmproxy.net import tcp +from mitmproxy.net.http import authentication +from mitmproxy.net.http import url CONF_BASENAME = "mitmproxy" @@ -106,7 +106,7 @@ class ProxyConfig: "Certificate Authority parent directory does not exist: %s" % os.path.dirname(options.cadir) ) - self.certstore = certutils.CertStore.from_store( + self.certstore = certs.CertStore.from_store( certstore_path, CONF_BASENAME ) diff --git a/mitmproxy/proxy/modes/socks_proxy.py b/mitmproxy/proxy/modes/socks_proxy.py index 4f8473bd..adcd8fc1 100644 --- a/mitmproxy/proxy/modes/socks_proxy.py +++ b/mitmproxy/proxy/modes/socks_proxy.py @@ -1,7 +1,6 @@ -import netlib.exceptions from mitmproxy import exceptions from mitmproxy.proxy import protocol -from netlib import socks +from mitmproxy.net import socks class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin): @@ -48,7 +47,7 @@ class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin): connect_reply.to_file(self.client_conn.wfile) self.client_conn.wfile.flush() - except (socks.SocksError, netlib.exceptions.TcpException) as e: + except (socks.SocksError, exceptions.TcpException) as e: raise exceptions.Socks5ProtocolException("SOCKS5 mode failure: %s" % repr(e)) self.server_conn.address = connect_request.addr diff --git a/mitmproxy/proxy/protocol/base.py b/mitmproxy/proxy/protocol/base.py index 00d50721..97e90051 100644 --- a/mitmproxy/proxy/protocol/base.py +++ b/mitmproxy/proxy/protocol/base.py @@ -1,4 +1,3 @@ -import netlib.exceptions from mitmproxy import exceptions from mitmproxy import connections @@ -177,7 +176,7 @@ class ServerConnectionMixin: self.channel.ask("serverconnect", self.server_conn) try: self.server_conn.connect() - except netlib.exceptions.TcpException as e: + except exceptions.TcpException as e: raise exceptions.ProtocolException( "Server connection to {} failed: {}".format( repr(self.server_conn.address), str(e) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index ec018f89..a47fb455 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -1,5 +1,4 @@ import h2.exceptions -import netlib.exceptions import time import traceback from mitmproxy import exceptions @@ -7,9 +6,9 @@ from mitmproxy import http from mitmproxy import flow from mitmproxy.proxy.protocol import base from mitmproxy.proxy.protocol import websockets as pwebsockets -import netlib.http -from netlib import tcp -from netlib import websockets +import mitmproxy.net.http +from mitmproxy.net import tcp +from mitmproxy.net import websockets class _HttpTransmissionLayer(base.Layer): @@ -46,7 +45,7 @@ class _HttpTransmissionLayer(base.Layer): def send_response(self, response): if response.data.content is None: - raise netlib.exceptions.HttpException("Cannot assemble flow with missing content") + raise exceptions.HttpException("Cannot assemble flow with missing content") self.send_response_headers(response) self.send_response_body(response, [response.data.content]) @@ -146,10 +145,10 @@ class HttpLayer(base.Layer): request = self.get_request_from_client(f) # Make sure that the incoming request matches our expectations self.validate_request(request) - except netlib.exceptions.HttpReadDisconnect: + except exceptions.HttpReadDisconnect: # don't throw an error for disconnects that happen before/between requests. return - except netlib.exceptions.HttpException as e: + except exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end self.send_error_response(400, repr(e)) @@ -173,7 +172,7 @@ class HttpLayer(base.Layer): if self.mode == "regular" and request.first_line_format == "authority": self.handle_regular_mode_connect(request) return - except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: + except (exceptions.ProtocolException, exceptions.NetlibException) as e: # HTTPS tasting means that ordinary errors like resolution and # connection errors can happen here. self.send_error_response(502, repr(e)) @@ -224,7 +223,7 @@ class HttpLayer(base.Layer): self.handle_upstream_mode_connect(f.request.copy()) return - except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: + except (exceptions.ProtocolException, exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not f.response: f.error = flow.Error(str(e)) @@ -254,7 +253,7 @@ class HttpLayer(base.Layer): try: response = http.make_error_response(code, message, headers) self.send_response(response) - except (netlib.exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException): + except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException): self.log(traceback.format_exc(), "debug") def change_upstream_proxy_server(self, address): @@ -300,7 +299,7 @@ class HttpLayer(base.Layer): try: get_response() - except netlib.exceptions.NetlibException as e: + except exceptions.NetlibException as e: self.log( "server communication error: %s" % repr(e), level="debug" @@ -396,7 +395,7 @@ class HttpLayer(base.Layer): def validate_request(self, request): if request.first_line_format == "absolute" and request.scheme != "http": - raise netlib.exceptions.HttpException("Invalid request scheme: %s" % request.scheme) + raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { "regular": ("authority", "absolute",), @@ -409,7 +408,7 @@ class HttpLayer(base.Layer): err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( " or ".join(allowed_request_forms), request.first_line_format ) - raise netlib.exceptions.HttpException(err_message) + raise exceptions.HttpException(err_message) if self.mode == "regular" and request.first_line_format == "absolute": request.first_line_format = "relative" @@ -423,13 +422,13 @@ class HttpLayer(base.Layer): self.send_response(http.make_error_response( 401, "Authentication Required", - netlib.http.Headers(**self.config.authenticator.auth_challenge_headers()) + mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers()) )) else: self.send_response(http.make_error_response( 407, "Proxy Authentication Required", - netlib.http.Headers(**self.config.authenticator.auth_challenge_headers()) + mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers()) )) return False return True diff --git a/mitmproxy/proxy/protocol/http1.py b/mitmproxy/proxy/protocol/http1.py index 968d3249..713c48a7 100644 --- a/mitmproxy/proxy/protocol/http1.py +++ b/mitmproxy/proxy/protocol/http1.py @@ -1,6 +1,6 @@ from mitmproxy import http from mitmproxy.proxy.protocol import http as httpbase -from netlib.http import http1 +from mitmproxy.net.http import http1 class Http1Layer(httpbase._HttpTransmissionLayer): diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index cbd8b34c..f635e710 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -8,15 +8,14 @@ from h2 import connection from h2 import events import queue -import netlib.exceptions from mitmproxy import exceptions from mitmproxy import http from mitmproxy.proxy.protocol import base from mitmproxy.proxy.protocol import http as httpbase -import netlib.http -from netlib import tcp -from netlib import basethread -from netlib.http import http2 +import mitmproxy.net.http +from mitmproxy.net import tcp +from mitmproxy.types import basethread +from mitmproxy.net.http import http2 class SafeH2Connection(connection.H2Connection): @@ -149,7 +148,7 @@ class Http2Layer(base.Layer): return True def _handle_request_received(self, eid, event, h2_connection): - headers = netlib.http.Headers([[k, v] for k, v in event.headers]) + headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, h2_connection, eid, headers) self.streams[eid].timestamp_start = time.time() self.streams[eid].no_body = (event.stream_ended is not None) @@ -163,7 +162,7 @@ class Http2Layer(base.Layer): return True def _handle_response_received(self, eid, event): - headers = netlib.http.Headers([[k, v] for k, v in event.headers]) + headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid].queued_data_length = 0 self.streams[eid].timestamp_start = time.time() self.streams[eid].response_headers = headers @@ -240,7 +239,7 @@ class Http2Layer(base.Layer): self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers) self.client_conn.send(self.client_conn.h2.data_to_send()) - headers = netlib.http.Headers([[k, v] for k, v in event.headers]) + headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, h2_connection, event.pushed_stream_id, headers) self.streams[event.pushed_stream_id].timestamp_start = time.time() self.streams[event.pushed_stream_id].pushed = True diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index bf0697be..c37badd3 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -1,14 +1,13 @@ import traceback -import netlib.exceptions from mitmproxy import log from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import http from mitmproxy import flow from mitmproxy import connections -from netlib.http import http1 -from netlib import basethread +from mitmproxy.net.http import http1 +from mitmproxy.types import basethread # TODO: Doesn't really belong into mitmproxy.proxy.protocol... @@ -97,7 +96,7 @@ class RequestReplayThread(basethread.BaseThread): response_reply = self.channel.ask("response", self.f) if response_reply == exceptions.Kill: raise exceptions.Kill() - except (exceptions.ReplayException, netlib.exceptions.NetlibException) as e: + except (exceptions.ReplayException, exceptions.NetlibException) as e: self.f.error = flow.Error(str(e)) if self.channel: self.channel.ask("error", self.f) diff --git a/mitmproxy/proxy/protocol/rawtcp.py b/mitmproxy/proxy/protocol/rawtcp.py index 513b90b3..0ec50594 100644 --- a/mitmproxy/proxy/protocol/rawtcp.py +++ b/mitmproxy/proxy/protocol/rawtcp.py @@ -2,10 +2,10 @@ import socket from OpenSSL import SSL -import netlib.exceptions -import netlib.tcp +import mitmproxy.net.tcp from mitmproxy import tcp from mitmproxy import flow +from mitmproxy import exceptions from mitmproxy.proxy.protocol import base @@ -31,7 +31,7 @@ class RawTCPLayer(base.Layer): try: while not self.channel.should_exit.is_set(): - r = netlib.tcp.ssl_read_select(conns, 10) + r = mitmproxy.net.tcp.ssl_read_select(conns, 10) for conn in r: dst = server if conn == client else client @@ -56,7 +56,7 @@ class RawTCPLayer(base.Layer): self.channel.ask("tcp_message", f) dst.sendall(tcp_message.content) - except (socket.error, netlib.exceptions.TcpException, SSL.Error) as e: + except (socket.error, exceptions.TcpException, SSL.Error) as e: if not self.ignore: f.error = flow.Error("TCP connection closed unexpectedly: {}".format(repr(e))) self.channel.tell("tcp_error", f) diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 940ab9ea..8a344faf 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -3,11 +3,10 @@ from typing import Optional # noqa from typing import Union import construct -import netlib.exceptions from mitmproxy import exceptions from mitmproxy.contrib.tls import _constructs from mitmproxy.proxy.protocol import base -from netlib import utils +from mitmproxy.net import check # taken from https://testssl.sh/openssl-rfc.mappping.html @@ -265,7 +264,7 @@ class TlsClientHello: extension.type == 0x00 and len(extension.server_names) == 1 and extension.server_names[0].type == 0 and - utils.is_valid_host(extension.server_names[0].name) + check.is_valid_host(extension.server_names[0].name) ) if is_valid_sni_extension: return extension.server_names[0].name.decode("idna") @@ -484,7 +483,7 @@ class TlsLayer(base.Layer): # The reason for this might be difficult to find, so we try to peek here to see if it # raises ann error. self.client_conn.rfile.peek(1) - except netlib.exceptions.TlsException as e: + except exceptions.TlsException as e: raise exceptions.ClientHandshakeException( "Cannot establish TLS with client (sni: {sni}): {e}".format( sni=self._client_hello.sni, e=repr(e) @@ -528,9 +527,9 @@ class TlsLayer(base.Layer): if tls_cert_err is not None: self.log(str(tls_cert_err), "warn") self.log("Ignoring server verification error, continuing with connection", "warn") - except netlib.exceptions.InvalidCertificateException as e: + except exceptions.InvalidCertificateException as e: raise exceptions.InvalidServerCertificate(str(e)) - except netlib.exceptions.TlsException as e: + except exceptions.TlsException as e: raise exceptions.TlsProtocolException( "Cannot establish TLS with {address} (sni: {sni}): {e}".format( address=repr(self.server_conn.address), diff --git a/mitmproxy/proxy/protocol/websockets.py b/mitmproxy/proxy/protocol/websockets.py index 816ec92d..ca1d05cb 100644 --- a/mitmproxy/proxy/protocol/websockets.py +++ b/mitmproxy/proxy/protocol/websockets.py @@ -1,12 +1,11 @@ -import netlib.exceptions import socket import struct from OpenSSL import SSL from mitmproxy import exceptions from mitmproxy.proxy.protocol import base -from netlib import strutils -from netlib import tcp -from netlib import websockets +from mitmproxy.utils import strutils +from mitmproxy.net import tcp +from mitmproxy.net import websockets class WebSocketsLayer(base.Layer): @@ -105,7 +104,7 @@ class WebSocketsLayer(base.Layer): if not self._handle_frame(frame, source_conn, other_conn, is_server): return - except (socket.error, netlib.exceptions.TcpException, SSL.Error) as e: + except (socket.error, exceptions.TcpException, SSL.Error) as e: self.log("WebSockets connection closed unexpectedly by {}: {}".format( "server" if is_server else "client", repr(e)), "info") except Exception as e: # pragma: no cover diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index 6a99d9cf..eacf7e0b 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -1,4 +1,3 @@ -import netlib.exceptions from mitmproxy import log from mitmproxy import exceptions from mitmproxy.proxy import protocol @@ -43,7 +42,7 @@ class RootContext: def _next_layer(self, top_layer): try: d = top_layer.client_conn.rfile.peek(3) - except netlib.exceptions.TcpException as e: + except exceptions.TcpException as e: raise exceptions.ProtocolException(str(e)) client_tls = protocol.is_tls_record_magic(d) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index b876f9ce..232d7038 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -2,15 +2,14 @@ import socket import sys import traceback -import netlib.exceptions from mitmproxy import exceptions from mitmproxy import connections from mitmproxy import http from mitmproxy import log from mitmproxy.proxy import modes from mitmproxy.proxy import root_context -from netlib import tcp -from netlib.http import http1 +from mitmproxy.net import tcp +from mitmproxy.net.http import http1 class DummyServer: @@ -138,7 +137,7 @@ class ConnectionHandler: try: error_response = http.make_error_response(502, repr(e)) self.client_conn.send(http1.assemble_response(error_response)) - except netlib.exceptions.TcpException: + except exceptions.TcpException: pass except Exception: self.log(traceback.format_exc(), "error") diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py index dc72e5b7..2fd7ad8d 100644 --- a/mitmproxy/script/concurrent.py +++ b/mitmproxy/script/concurrent.py @@ -4,7 +4,7 @@ offload computations from mitmproxy's main master thread. """ from mitmproxy import events -from netlib import basethread +from mitmproxy.types import basethread class ScriptThread(basethread.BaseThread): diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index f4415ecf..1ab744a5 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,7 +1,7 @@ from typing import Any from typing import List -import netlib.basetypes +from mitmproxy.types import serializable def _is_list(cls): @@ -10,7 +10,7 @@ def _is_list(cls): return issubclass(cls, List) or is_list_bugfix -class StateObject(netlib.basetypes.Serializable): +class StateObject(serializable.Serializable): """ An object with serializable state. diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index af54c9d4..d73be98d 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -2,11 +2,11 @@ import time from typing import List -import netlib.basetypes from mitmproxy import flow +from mitmproxy.types import serializable -class TCPMessage(netlib.basetypes.Serializable): +class TCPMessage(serializable.Serializable): def __init__(self, from_client, content, timestamp=None): self.content = content diff --git a/netlib/tutils.py b/mitmproxy/test/tutils.py index d22fdd1c..e925b282 100644 --- a/netlib/tutils.py +++ b/mitmproxy/test/tutils.py @@ -6,7 +6,9 @@ import shutil from contextlib import contextmanager import sys -from netlib import utils, tcp, http +from mitmproxy.utils import data +from mitmproxy.net import tcp +from mitmproxy.net import http def treader(bytes): @@ -87,15 +89,13 @@ class RaisesContext: return True -test_data = utils.Data(__name__) -# FIXME: Temporary workaround during repo merge. -test_data.dirname = os.path.join(test_data.dirname, "..", "test", "netlib") +test_data = data.Data(__name__).push("../../test/mitmproxy/net") def treq(**kwargs): """ Returns: - netlib.http.Request + mitmproxy.net.http.Request """ default = dict( first_line_format="relative", @@ -115,7 +115,7 @@ def treq(**kwargs): def tresp(**kwargs): """ Returns: - netlib.http.Response + mitmproxy.net.http.Response """ default = dict( http_version=b"HTTP/1.1", diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ddc4a5ce..2f9ea15c 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -5,9 +5,9 @@ from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import options from mitmproxy import platform -from netlib import human -from netlib import tcp -from netlib import version +from mitmproxy.utils import human +from mitmproxy.net import tcp +from mitmproxy import version class ParseException(Exception): diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index d10d9321..90dce5ae 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -6,11 +6,11 @@ import os import urwid import urwid.util -import netlib -from mitmproxy import utils +import mitmproxy.net +from mitmproxy.utils import lrucache from mitmproxy.tools.console import signals from mitmproxy import export -from netlib import human +from mitmproxy.utils import human try: import pyperclip @@ -226,7 +226,7 @@ def format_flow_data(key, scope, flow): if request.content is None: return None, "Request content is missing" if key == "h": - data += netlib.http.http1.assemble_request(request) + data += mitmproxy.net.http.http1.assemble_request(request) elif key == "c": data += request.get_content(strict=False) else: @@ -240,7 +240,7 @@ def format_flow_data(key, scope, flow): if response.content is None: return None, "Response content is missing" if key == "h": - data += netlib.http.http1.assemble_response(response) + data += mitmproxy.net.http.http1.assemble_response(response) elif key == "c": data += response.get_content(strict=False) else: @@ -325,7 +325,7 @@ def export_to_clip_or_file(key, scope, flow, writer): else: # other keys writer(exporter(flow)) -flowcache = utils.LRUCache(800) +flowcache = lrucache.LRUCache(800) def raw_format_flow(f): diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index f13f9a1d..7591c3d1 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -1,7 +1,7 @@ import urwid from mitmproxy.tools.console import common, searchable -from netlib import human +from mitmproxy.utils import human def maybe_timestamp(base, attr): diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 31624229..d33348a0 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -1,6 +1,6 @@ import urwid -import netlib.http.url +import mitmproxy.net.http.url from mitmproxy import exceptions from mitmproxy.tools.console import common from mitmproxy.tools.console import signals @@ -325,7 +325,7 @@ class FlowListBox(urwid.ListBox): ) def new_request(self, url, method): - parts = netlib.http.url.parse(str(url)) + parts = mitmproxy.net.http.url.parse(str(url)) if not parts: signals.status_message.send(message="Invalid Url") return diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 64caf474..4e886ee4 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -8,7 +8,7 @@ from typing import Optional, Union # noqa from mitmproxy import contentviews from mitmproxy import http -from mitmproxy import utils +from mitmproxy.utils import lrucache from mitmproxy.tools.console import common from mitmproxy.tools.console import flowdetailview from mitmproxy.tools.console import grideditor @@ -16,8 +16,8 @@ from mitmproxy.tools.console import searchable from mitmproxy.tools.console import signals from mitmproxy.tools.console import tabs from mitmproxy import export -from netlib.http import Headers -from netlib.http import status_codes +from mitmproxy.net.http import Headers +from mitmproxy.net.http import status_codes class SearchError(Exception): @@ -121,7 +121,7 @@ class FlowViewHeader(urwid.WidgetWrap): ) -cache = utils.LRUCache(200) +cache = lrucache.LRUCache(200) TAB_REQ = 0 TAB_RESP = 1 diff --git a/mitmproxy/tools/console/grideditor/col_bytes.py b/mitmproxy/tools/console/grideditor/col_bytes.py index c951ce44..f580e947 100644 --- a/mitmproxy/tools/console/grideditor/col_bytes.py +++ b/mitmproxy/tools/console/grideditor/col_bytes.py @@ -4,7 +4,7 @@ from typing import Callable, Optional import urwid from mitmproxy.tools.console import signals from mitmproxy.tools.console.grideditor import base -from netlib import strutils +from mitmproxy.utils import strutils def read_file(filename: str, callback: Callable[..., None], escaped: bool) -> Optional[str]: diff --git a/mitmproxy/tools/console/grideditor/col_subgrid.py b/mitmproxy/tools/console/grideditor/col_subgrid.py index 3147e63d..4db37462 100644 --- a/mitmproxy/tools/console/grideditor/col_subgrid.py +++ b/mitmproxy/tools/console/grideditor/col_subgrid.py @@ -1,7 +1,7 @@ import urwid from mitmproxy.tools.console.grideditor import base from mitmproxy.tools.console import signals -from netlib.http import cookies +from mitmproxy.net.http import cookies class Column(base.Column): diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 64361af7..5e3f3d42 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -9,7 +9,7 @@ from mitmproxy.tools.console.grideditor import col_bytes from mitmproxy.tools.console.grideditor import col_text from mitmproxy.tools.console.grideditor import col_subgrid from mitmproxy.tools.console import signals -from netlib.http import user_agents +from mitmproxy.net.http import user_agents class QueryEditor(base.GridEditor): diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index 752ebf00..dda8bfbc 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -6,7 +6,7 @@ from mitmproxy import flowfilter from mitmproxy.tools.console import common from mitmproxy.tools.console import signals -from netlib import version +from mitmproxy import version footer = [ ("heading", 'mitmproxy {} (Python {}) '.format(version.VERSION, platform.python_version())), diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index b3fd0adb..af43c53b 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -21,7 +21,7 @@ from mitmproxy import exceptions from mitmproxy import master from mitmproxy import io from mitmproxy import flowfilter -from mitmproxy import utils +from mitmproxy import log from mitmproxy.addons import state import mitmproxy.options from mitmproxy.tools.console import flowlist @@ -35,7 +35,9 @@ from mitmproxy.tools.console import signals from mitmproxy.tools.console import statusbar from mitmproxy.tools.console import window from mitmproxy.flowfilter import FMarked -from netlib import tcp, strutils +from mitmproxy.utils import strutils + +from mitmproxy.net import tcp EVENTLOG_SIZE = 500 @@ -266,7 +268,7 @@ class ConsoleMaster(master.Master): ) def sig_add_log(self, sender, e, level): - if self.options.verbosity < utils.log_tier(level): + if self.options.verbosity < log.log_tier(level): return if level in ("error", "warn"): diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 99f73727..74382677 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -2,11 +2,11 @@ import os.path import urwid -import netlib.http.url +import mitmproxy.net.http.url from mitmproxy.tools.console import common from mitmproxy.tools.console import pathedit from mitmproxy.tools.console import signals -from netlib import human +from mitmproxy.utils import human class ActionBar(urwid.WidgetWrap): @@ -208,7 +208,7 @@ class StatusBar(urwid.WidgetWrap): if self.master.options.mode in ["reverse", "upstream"]: dst = self.master.server.config.upstream_server - r.append("[dest:%s]" % netlib.http.url.unparse( + r.append("[dest:%s]" % mitmproxy.net.http.url.unparse( dst.scheme, dst.address.host, dst.address.port diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 47f69303..e92482f3 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -8,7 +8,7 @@ from mitmproxy import io from mitmproxy import options from mitmproxy import master from mitmproxy.addons import dumper, termlog -from netlib import tcp +from mitmproxy.net import tcp class DumpError(Exception): diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 5d4e8dc9..9fce9234 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -6,8 +6,8 @@ from mitmproxy.tools import cmdline from mitmproxy import exceptions from mitmproxy.proxy import config from mitmproxy.proxy import server -from netlib import version_check -from netlib import debug +from mitmproxy.utils import version_check +from mitmproxy.utils import debug def assert_utf8_env(): diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 89b42186..41701d36 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -15,7 +15,7 @@ from mitmproxy import flow from mitmproxy import http from mitmproxy import contentviews from mitmproxy import io -from netlib import version +from mitmproxy import version def convert_flow_to_json_dict(flow: flow.Flow) -> dict: diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 2bca4555..619582f3 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -13,7 +13,7 @@ from mitmproxy.addons import state from mitmproxy import options from mitmproxy import master from mitmproxy.tools.web import app -from netlib.http import authentication +from mitmproxy.net.http import authentication class Stop(Exception): diff --git a/test/netlib/__init__.py b/mitmproxy/types/__init__.py index e69de29b..e69de29b 100644 --- a/test/netlib/__init__.py +++ b/mitmproxy/types/__init__.py diff --git a/netlib/basethread.py b/mitmproxy/types/basethread.py index a3c81d19..a3c81d19 100644 --- a/netlib/basethread.py +++ b/mitmproxy/types/basethread.py diff --git a/mitmproxy/types/bidi.py b/mitmproxy/types/bidi.py new file mode 100644 index 00000000..0982a34a --- /dev/null +++ b/mitmproxy/types/bidi.py @@ -0,0 +1,29 @@ + + +class BiDi: + + """ + A wee utility class for keeping bi-directional mappings, like field + constants in protocols. Names are attributes on the object, dict-like + access maps values to names: + + CONST = BiDi(a=1, b=2) + assert CONST.a == 1 + assert CONST.get_name(1) == "a" + """ + + def __init__(self, **kwargs): + self.names = kwargs + self.values = {} + for k, v in kwargs.items(): + self.values[v] = k + if len(self.names) != len(self.values): + raise ValueError("Duplicate values not allowed.") + + def __getattr__(self, k): + if k in self.names: + return self.names[k] + raise AttributeError("No such attribute: %s", k) + + def get_name(self, n, default=None): + return self.values.get(n, default) diff --git a/netlib/multidict.py b/mitmproxy/types/multidict.py index 191d1cc6..d351e48b 100644 --- a/netlib/multidict.py +++ b/mitmproxy/types/multidict.py @@ -6,10 +6,10 @@ try: except ImportError: # pragma: no cover from collections import MutableMapping # Workaround for Python < 3.3 -from netlib import basetypes +from mitmproxy.types import serializable -class _MultiDict(MutableMapping, basetypes.Serializable, metaclass=ABCMeta): +class _MultiDict(MutableMapping, serializable.Serializable, metaclass=ABCMeta): def __repr__(self): fields = ( repr(field) diff --git a/netlib/basetypes.py b/mitmproxy/types/serializable.py index 49892ffc..49892ffc 100644 --- a/netlib/basetypes.py +++ b/mitmproxy/types/serializable.py diff --git a/test/netlib/http/__init__.py b/mitmproxy/utils/__init__.py index e69de29b..e69de29b 100644 --- a/test/netlib/http/__init__.py +++ b/mitmproxy/utils/__init__.py diff --git a/mitmproxy/utils/bits.py b/mitmproxy/utils/bits.py new file mode 100644 index 00000000..ec0d40ef --- /dev/null +++ b/mitmproxy/utils/bits.py @@ -0,0 +1,15 @@ + + +def setbit(byte, offset, value): + """ + Set a bit in a byte to 1 if value is truthy, 0 if not. + """ + if value: + return byte | (1 << offset) + else: + return byte & ~(1 << offset) + + +def getbit(byte, offset): + mask = 1 << offset + return bool(byte & mask) diff --git a/mitmproxy/utils/data.py b/mitmproxy/utils/data.py new file mode 100644 index 00000000..2e68d184 --- /dev/null +++ b/mitmproxy/utils/data.py @@ -0,0 +1,33 @@ +import os.path +import importlib +import inspect + + +class Data: + + def __init__(self, name): + m = importlib.import_module(name) + dirname = os.path.dirname(inspect.getsourcefile(m)) + self.dirname = os.path.abspath(dirname) + + def push(self, subpath): + """ + Change the data object to a path relative to the module. + """ + self.dirname = os.path.join(self.dirname, subpath) + return self + + def path(self, path): + """ + Returns a path to the package data housed at 'path' under this + module.Path can be a path to a file, or to a directory. + + This function will raise ValueError if the path does not exist. + """ + fullpath = os.path.join(self.dirname, path) + if not os.path.exists(fullpath): + raise ValueError("dataPath: %s does not exist." % fullpath) + return fullpath + + +pkg_data = Data(__name__).push("..") diff --git a/netlib/debug.py b/mitmproxy/utils/debug.py index f1b3d792..ac8fedd7 100644 --- a/netlib/debug.py +++ b/mitmproxy/utils/debug.py @@ -6,7 +6,7 @@ import signal import platform import traceback -from netlib import version +from mitmproxy import version from OpenSSL import SSL @@ -84,7 +84,7 @@ def dump_info(signal=None, frame=None, file=sys.stdout, testing=False): # pragm d = {} for i in gc.get_objects(): t = str(type(i)) - if "mitmproxy" in t or "netlib" in t: + if "mitmproxy" in t: d[t] = d.setdefault(t, 0) + 1 itms = list(d.items()) itms.sort(key=lambda x: x[1]) diff --git a/netlib/human.py b/mitmproxy/utils/human.py index 72e96d30..72e96d30 100644 --- a/netlib/human.py +++ b/mitmproxy/utils/human.py diff --git a/mitmproxy/utils.py b/mitmproxy/utils/lrucache.py index fb2effd8..7ad2b7f5 100644 --- a/mitmproxy/utils.py +++ b/mitmproxy/utils/lrucache.py @@ -1,7 +1,3 @@ -import netlib.utils - - -pkg_data = netlib.utils.Data(__name__) class LRUCache: @@ -34,7 +30,3 @@ class LRUCache: d = self.cacheList.pop() self.cache.pop(d) return ret - - -def log_tier(level): - return dict(error=0, warn=1, info=2, debug=3).get(level) diff --git a/netlib/strutils.py b/mitmproxy/utils/strutils.py index 57cfbc79..57cfbc79 100644 --- a/netlib/strutils.py +++ b/mitmproxy/utils/strutils.py diff --git a/netlib/version_check.py b/mitmproxy/utils/version_check.py index 547c031c..4cf2b9e6 100644 --- a/netlib/version_check.py +++ b/mitmproxy/utils/version_check.py @@ -1,7 +1,6 @@ """ -Having installed a wrong version of pyOpenSSL or netlib is unfortunately a -very common source of error. Check before every start that both versions -are somewhat okay. +Having installed a wrong version of pyOpenSSL is unfortunately a very common +source of error. Check before every start that both versions are somewhat okay. """ import sys import inspect diff --git a/netlib/version.py b/mitmproxy/version.py index cb670642..cb670642 100644 --- a/netlib/version.py +++ b/mitmproxy/version.py diff --git a/netlib/exceptions.py b/netlib/exceptions.py deleted file mode 100644 index d0b15d27..00000000 --- a/netlib/exceptions.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -We try to be very hygienic regarding the exceptions we throw: -Every Exception netlib raises shall be a subclass of NetlibException. - - -See also: http://lucumr.pocoo.org/2014/10/16/on-error-handling/ -""" - - -class NetlibException(Exception): - """ - Base class for all exceptions thrown by netlib. - """ - def __init__(self, message=None): - super().__init__(message) - - -class Disconnect: - """Immediate EOF""" - - -class HttpException(NetlibException): - pass - - -class HttpReadDisconnect(HttpException, Disconnect): - pass - - -class HttpSyntaxException(HttpException): - pass - - -class TcpException(NetlibException): - pass - - -class TcpDisconnect(TcpException, Disconnect): - pass - - -class TcpReadIncomplete(TcpException): - pass - - -class TcpTimeout(TcpException): - pass - - -class TlsException(NetlibException): - pass - - -class InvalidCertificateException(TlsException): - pass - - -class Timeout(TcpException): - pass diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py deleted file mode 100644 index 315f61ac..00000000 --- a/netlib/http/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from netlib.http.request import Request -from netlib.http.response import Response -from netlib.http.message import Message -from netlib.http.headers import Headers, parse_content_type -from netlib.http.message import decoded -from netlib.http import http1, http2, status_codes, multipart - -__all__ = [ - "Request", - "Response", - "Message", - "Headers", "parse_content_type", - "decoded", - "http1", "http2", "status_codes", "multipart", -] diff --git a/netlib/http/http2/__init__.py b/netlib/http/http2/__init__.py deleted file mode 100644 index 20cc63a0..00000000 --- a/netlib/http/http2/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from netlib.http.http2.framereader import read_raw_frame, parse_frame -from netlib.http.http2.utils import parse_headers - -__all__ = [ - "read_raw_frame", - "parse_frame", - "parse_headers", -] diff --git a/netlib/utils.py b/netlib/utils.py deleted file mode 100644 index 8cd9ba6e..00000000 --- a/netlib/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -import os.path -import re -import importlib -import inspect - - -def setbit(byte, offset, value): - """ - Set a bit in a byte to 1 if value is truthy, 0 if not. - """ - if value: - return byte | (1 << offset) - else: - return byte & ~(1 << offset) - - -def getbit(byte, offset): - mask = 1 << offset - return bool(byte & mask) - - -class BiDi: - - """ - A wee utility class for keeping bi-directional mappings, like field - constants in protocols. Names are attributes on the object, dict-like - access maps values to names: - - CONST = BiDi(a=1, b=2) - assert CONST.a == 1 - assert CONST.get_name(1) == "a" - """ - - def __init__(self, **kwargs): - self.names = kwargs - self.values = {} - for k, v in kwargs.items(): - self.values[v] = k - if len(self.names) != len(self.values): - raise ValueError("Duplicate values not allowed.") - - def __getattr__(self, k): - if k in self.names: - return self.names[k] - raise AttributeError("No such attribute: %s", k) - - def get_name(self, n, default=None): - return self.values.get(n, default) - - -class Data: - - def __init__(self, name): - m = importlib.import_module(name) - dirname = os.path.dirname(inspect.getsourcefile(m)) - self.dirname = os.path.abspath(dirname) - - def push(self, subpath): - """ - Change the data object to a path relative to the module. - """ - self.dirname = os.path.join(self.dirname, subpath) - return self - - def path(self, path): - """ - Returns a path to the package data housed at 'path' under this - module.Path can be a path to a file, or to a directory. - - This function will raise ValueError if the path does not exist. - """ - fullpath = os.path.join(self.dirname, path) - if not os.path.exists(fullpath): - raise ValueError("dataPath: %s does not exist." % fullpath) - return fullpath - - -_label_valid = re.compile(b"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE) - - -def is_valid_host(host: bytes) -> bool: - """ - Checks if a hostname is valid. - """ - try: - host.decode("idna") - except ValueError: - return False - if len(host) > 255: - return False - if host and host[-1:] == b".": - host = host[:-1] - return all(_label_valid.match(x) for x in host.split(b".")) - - -def is_valid_port(port): - return 0 <= port <= 65535 diff --git a/pathod/language/base.py b/pathod/language/base.py index 11f0899d..44a888c0 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -4,8 +4,8 @@ import abc import functools import pyparsing as pp -from netlib import strutils -from netlib import human +from mitmproxy.utils import strutils +from mitmproxy.utils import human from . import generators, exceptions diff --git a/pathod/language/http.py b/pathod/language/http.py index 32f990bb..8fcf9edc 100644 --- a/pathod/language/http.py +++ b/pathod/language/http.py @@ -2,12 +2,12 @@ import abc import pyparsing as pp -from netlib.http import url -import netlib.websockets -from netlib.http import status_codes, user_agents +from mitmproxy.net.http import url +import mitmproxy.net.websockets +from mitmproxy.net.http import status_codes, user_agents from . import base, exceptions, actions, message -# TODO: use netlib.semantics.protocol assemble method, +# TODO: use mitmproxy.net.semantics.protocol assemble method, # instead of duplicating the HTTP on-the-wire representation here. # see http2 language for an example @@ -198,7 +198,7 @@ class Response(_HTTPMessage): 1, StatusCode(101) ) - headers = netlib.websockets.server_handshake_headers( + headers = mitmproxy.net.websockets.server_handshake_headers( settings.websocket_key ) for i in headers.fields: @@ -310,7 +310,7 @@ class Request(_HTTPMessage): 1, Method("get") ) - for i in netlib.websockets.client_handshake_headers().fields: + for i in mitmproxy.net.websockets.client_handshake_headers().fields: if not get_header(i[0], self.headers): tokens.append( Header( diff --git a/pathod/language/http2.py b/pathod/language/http2.py index 35fc5ba8..08c5f6d7 100644 --- a/pathod/language/http2.py +++ b/pathod/language/http2.py @@ -1,7 +1,7 @@ import pyparsing as pp -from netlib import http -from netlib.http import user_agents, Headers +from mitmproxy.net import http +from mitmproxy.net.http import user_agents, Headers from . import base, message """ diff --git a/pathod/language/message.py b/pathod/language/message.py index 03b4a2cf..6cdaaa0b 100644 --- a/pathod/language/message.py +++ b/pathod/language/message.py @@ -1,6 +1,6 @@ import abc from . import actions, exceptions -from netlib import strutils +from mitmproxy.utils import strutils LOG_TRUNCATE = 1024 diff --git a/pathod/language/websockets.py b/pathod/language/websockets.py index 417944af..a237381c 100644 --- a/pathod/language/websockets.py +++ b/pathod/language/websockets.py @@ -1,7 +1,7 @@ import random import string -import netlib.websockets -from netlib import strutils +import mitmproxy.net.websockets +from mitmproxy.utils import strutils import pyparsing as pp from . import base, generators, actions, message @@ -14,12 +14,12 @@ class WF(base.CaselessLiteral): class OpCode(base.IntField): names = { - "continue": netlib.websockets.OPCODE.CONTINUE, - "text": netlib.websockets.OPCODE.TEXT, - "binary": netlib.websockets.OPCODE.BINARY, - "close": netlib.websockets.OPCODE.CLOSE, - "ping": netlib.websockets.OPCODE.PING, - "pong": netlib.websockets.OPCODE.PONG, + "continue": mitmproxy.net.websockets.OPCODE.CONTINUE, + "text": mitmproxy.net.websockets.OPCODE.TEXT, + "binary": mitmproxy.net.websockets.OPCODE.BINARY, + "close": mitmproxy.net.websockets.OPCODE.CLOSE, + "ping": mitmproxy.net.websockets.OPCODE.PING, + "pong": mitmproxy.net.websockets.OPCODE.PONG, } max = 15 preamble = "c" @@ -215,11 +215,11 @@ class WebsocketFrame(message.Message): v = getattr(self, i, None) if v is not None: frameparts[i] = v.value - frame = netlib.websockets.FrameHeader(**frameparts) + frame = mitmproxy.net.websockets.FrameHeader(**frameparts) vals = [bytes(frame)] if bodygen: if frame.masking_key and not self.rawbody: - masker = netlib.websockets.Masker(frame.masking_key) + masker = mitmproxy.net.websockets.Masker(frame.masking_key) vals.append( generators.TransformGenerator( bodygen, diff --git a/pathod/language/writer.py b/pathod/language/writer.py index b8081989..ac0f44da 100644 --- a/pathod/language/writer.py +++ b/pathod/language/writer.py @@ -1,5 +1,5 @@ import time -from netlib.exceptions import TcpDisconnect +from mitmproxy import exceptions BLOCKSIZE = 1024 # It's not clear what the upper limit for time.sleep is. It's lower than the @@ -62,5 +62,5 @@ def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE): return True elif a[1] == "inject": send_chunk(fp, a[2], blocksize, 0, len(a[2])) - except TcpDisconnect: # pragma: no cover + except exceptions.TcpDisconnect: # pragma: no cover return True diff --git a/pathod/log.py b/pathod/log.py index 37100bb1..4e5f355f 100644 --- a/pathod/log.py +++ b/pathod/log.py @@ -1,6 +1,7 @@ import time -from netlib import strutils, human +from mitmproxy.utils import strutils +from mitmproxy.utils import human def write_raw(fp, lines, timestamp=True): diff --git a/pathod/pathoc.py b/pathod/pathoc.py index e9fa5c43..066c330c 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -11,15 +11,19 @@ import time import OpenSSL.crypto import logging -from netlib.tutils import treq -from netlib import strutils -from netlib import tcp, certutils, websockets, socks -from netlib import exceptions -from netlib.http import http1 -from netlib import basethread +from mitmproxy.test.tutils import treq +from mitmproxy.utils import strutils +from mitmproxy.net import tcp +from mitmproxy import certs +from mitmproxy.net import websockets +from mitmproxy.net import socks +from mitmproxy import exceptions +from mitmproxy.net.http import http1 +from mitmproxy.types import basethread -from . import log, language -from .protocols import http2 +from pathod import log +from pathod import language +from pathod.protocols import http2 logging.getLogger("hpack").setLevel(logging.WARNING) @@ -76,7 +80,7 @@ class SSLInfo: } t = types.get(pk.type(), "Uknown") parts.append("\tPubkey: %s bit %s" % (pk.bits(), t)) - s = certutils.SSLCert(i) + s = certs.SSLCert(i) if s.altnames: parts.append("\tSANs: %s" % " ".join(strutils.native(n, "utf8") for n in s.altnames)) return "\n".join(parts) diff --git a/pathod/pathoc_cmdline.py b/pathod/pathoc_cmdline.py index ab330505..3b738d47 100644 --- a/pathod/pathoc_cmdline.py +++ b/pathod/pathoc_cmdline.py @@ -3,9 +3,9 @@ import argparse import os import os.path -from netlib import tcp -from netlib import version -from netlib.http import user_agents +from mitmproxy.net import tcp +from mitmproxy import version +from mitmproxy.net.http import user_agents from . import pathoc, language diff --git a/pathod/pathod.py b/pathod/pathod.py index 0f659d40..a8658361 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -4,16 +4,18 @@ import os import sys import threading -from netlib import tcp -from netlib import certutils -from netlib import websockets -from netlib import version +from mitmproxy.net import tcp +from mitmproxy import certs as mcerts +from mitmproxy.net import websockets +from mitmproxy import version import urllib -from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \ - TlsException +from mitmproxy import exceptions -from . import language, utils, log, protocols +from pathod import language +from pathod import utils +from pathod import log +from pathod import protocols DEFAULT_CERT_DOMAIN = b"pathod.net" @@ -52,7 +54,7 @@ class SSLOptions: self.ssl_options = ssl_options self.ciphers = ciphers self.alpn_select = alpn_select - self.certstore = certutils.CertStore.from_store( + self.certstore = mcerts.CertStore.from_store( os.path.expanduser(confdir), CERTSTORE_BASENAME ) @@ -128,9 +130,9 @@ class PathodHandler(tcp.BaseHandler): with logger.ctx() as lg: try: req = self.protocol.read_request(self.rfile) - except HttpReadDisconnect: + except exceptions.HttpReadDisconnect: return None, None - except HttpException as s: + except exceptions.HttpException as s: s = str(s) lg(s) return None, dict(type="error", msg=s) @@ -252,7 +254,7 @@ class PathodHandler(tcp.BaseHandler): options=self.server.ssloptions.ssl_options, alpn_select=self.server.ssloptions.alpn_select, ) - except TlsException as v: + except exceptions.TlsException as v: s = str(v) self.server.add_log( dict( @@ -384,7 +386,7 @@ class Pathod(tcp.TCPServer): try: h.handle() h.finish() - except TcpDisconnect: # pragma: no cover + except exceptions.TcpDisconnect: # pragma: no cover log.write_raw(self.logfp, "Disconnect") self.add_log( dict( @@ -393,7 +395,7 @@ class Pathod(tcp.TCPServer): ) ) return - except TcpTimeout: + except exceptions.TcpTimeout: log.write_raw(self.logfp, "Timeout") self.add_log( dict( diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py index 3531b8f3..ef1e983f 100644 --- a/pathod/pathod_cmdline.py +++ b/pathod/pathod_cmdline.py @@ -4,9 +4,9 @@ import os import os.path import re -from netlib import tcp -from netlib import human -from netlib import version +from mitmproxy.net import tcp +from mitmproxy.utils import human +from mitmproxy import version from . import pathod diff --git a/pathod/protocols/http.py b/pathod/protocols/http.py index 17930320..4387b4fb 100644 --- a/pathod/protocols/http.py +++ b/pathod/protocols/http.py @@ -1,6 +1,6 @@ -from netlib import version -from netlib.exceptions import TlsException -from netlib.http import http1 +from mitmproxy import version +from mitmproxy import exceptions +from mitmproxy.net.http import http1 from .. import language @@ -37,7 +37,7 @@ class HTTPProtocol: options=self.pathod_handler.server.ssloptions.ssl_options, alpn_select=self.pathod_handler.server.ssloptions.alpn_select, ) - except TlsException as v: + except exceptions.TlsException as v: s = str(v) lg(s) return None, dict(type="error", msg=s) diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py index 4dcc4b76..118163d2 100644 --- a/pathod/protocols/http2.py +++ b/pathod/protocols/http2.py @@ -4,11 +4,11 @@ import time import hyperframe.frame from hpack.hpack import Encoder, Decoder -from netlib import utils -from netlib.http import http2 -import netlib.http.headers -import netlib.http.response -import netlib.http.request +from mitmproxy.net.http import http2 +import mitmproxy.net.http.headers +import mitmproxy.net.http.response +import mitmproxy.net.http.request +from mitmproxy.types import bidi from .. import language @@ -22,7 +22,7 @@ class TCPHandler: class HTTP2StateProtocol: - ERROR_CODES = utils.BiDi( + ERROR_CODES = bidi.BiDi( NO_ERROR=0x0, PROTOCOL_ERROR=0x1, INTERNAL_ERROR=0x2, @@ -100,7 +100,7 @@ class HTTP2StateProtocol: first_line_format, method, scheme, host, port, path = http2.parse_headers(headers) - request = netlib.http.request.Request( + request = mitmproxy.net.http.request.Request( first_line_format, method, scheme, @@ -148,7 +148,7 @@ class HTTP2StateProtocol: else: timestamp_end = None - response = netlib.http.response.Response( + response = mitmproxy.net.http.response.Response( b"HTTP/2.0", int(headers.get(':status', 502)), b'', @@ -162,15 +162,15 @@ class HTTP2StateProtocol: return response def assemble(self, message): - if isinstance(message, netlib.http.request.Request): + if isinstance(message, mitmproxy.net.http.request.Request): return self.assemble_request(message) - elif isinstance(message, netlib.http.response.Response): + elif isinstance(message, mitmproxy.net.http.response.Response): return self.assemble_response(message) else: raise ValueError("HTTP message not supported.") def assemble_request(self, request): - assert isinstance(request, netlib.http.request.Request) + assert isinstance(request, mitmproxy.net.http.request.Request) authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host if self.tcp_handler.address.port != 443: @@ -194,7 +194,7 @@ class HTTP2StateProtocol: self._create_body(request.body, stream_id))) def assemble_response(self, response): - assert isinstance(response, netlib.http.response.Response) + assert isinstance(response, mitmproxy.net.http.response.Response) headers = response.headers.copy() @@ -394,7 +394,7 @@ class HTTP2StateProtocol: else: self._handle_unexpected_frame(frm) - headers = netlib.http.headers.Headers( + headers = mitmproxy.net.http.headers.Headers( [[k, v] for k, v in self.decoder.decode(header_blocks, raw=True)] ) diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index df83461a..00ae5aa8 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -1,8 +1,8 @@ import time -from netlib import websockets -from .. import language -from netlib.exceptions import NetlibException +from mitmproxy.net import websockets +from pathod import language +from mitmproxy import exceptions class WebsocketsProtocol: @@ -16,7 +16,7 @@ class WebsocketsProtocol: started = time.time() try: frm = websockets.Frame.from_file(self.pathod_handler.rfile) - except NetlibException as e: + except exceptions.NetlibException as e: lg("Error reading websocket frame: %s" % e) return None, None ended = time.time() diff --git a/pathod/test.py b/pathod/test.py index c92cc50b..b819d723 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -3,7 +3,7 @@ import time import queue from . import pathod -from netlib import basethread +from mitmproxy.types import basethread class Daemon: diff --git a/pathod/utils.py b/pathod/utils.py index 2d077c48..44ad1f87 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -1,6 +1,6 @@ import os import sys -import netlib.utils +from mitmproxy.utils import data as mdata class MemBool: @@ -17,7 +17,8 @@ class MemBool: return bool(v) -data = netlib.utils.Data(__name__) +# FIXME: change this name +data = mdata.Data(__name__) def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover diff --git a/release/README.mkd b/release/README.mkd index db2ef970..c5505431 100644 --- a/release/README.mkd +++ b/release/README.mkd @@ -7,7 +7,7 @@ - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. This matches with what we do in `setup.py`: - "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION) + "mitmproxy>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION) # Release Checklist diff --git a/release/rtool.py b/release/rtool.py index 3ed9157e..fbca40e2 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -48,7 +48,7 @@ VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") # Project Configuration -VERSION_FILE = join(ROOT_DIR, "netlib", "version.py") +VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") PROJECT_NAME = "mitmproxy" PYTHON_VERSION = "py2.py3" BDISTS = { @@ -2,7 +2,7 @@ from setuptools import setup, find_packages from codecs import open import os -from netlib import version +from mitmproxy import version # Based on https://github.com/pypa/sampleproject/blob/master/setup.py # and https://python-packaging-user-guide.readthedocs.org/ @@ -44,7 +44,6 @@ setup( packages=find_packages(include=[ "mitmproxy", "mitmproxy.*", "pathod", "pathod.*", - "netlib", "netlib.*" ]), include_package_data=True, entry_points={ diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index e49f91bc..5f1b2dbd 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -7,7 +7,7 @@ from mitmproxy import exceptions from mitmproxy.tools import dump from mitmproxy import http from mitmproxy import proxy -import netlib.tutils +import mitmproxy.test.tutils import mock @@ -38,11 +38,11 @@ class TestDumper(mastertest.MasterTest): sio = io.StringIO() d.configure(dump.Options(tfile = sio, flow_detail = 4), updated) flow = tutils.tflow() - flow.request = netlib.tutils.treq() + flow.request = mitmproxy.test.tutils.treq() flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=None) + flow.response = mitmproxy.test.tutils.tresp(content=None) flow.response.is_replay = True flow.response.status_code = 300 d.response(flow) @@ -50,7 +50,7 @@ class TestDumper(mastertest.MasterTest): sio = io.StringIO() d.configure(dump.Options(tfile = sio, flow_detail = 4), updated) - flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) + flow = tutils.tflow(resp=mitmproxy.test.tutils.tresp(content=b"{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 d.response(flow) @@ -60,7 +60,7 @@ class TestDumper(mastertest.MasterTest): d.configure(dump.Options(tfile = sio), updated) flow = tutils.tflow() flow.request.content = None - flow.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + flow.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) flow.response.content = None d.response(flow) assert "content missing" in sio.getvalue() diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index 649b3c22..c8dd8704 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -1,6 +1,6 @@ from .. import tutils, mastertest -import netlib.tutils +import mitmproxy.test.tutils from mitmproxy.addons import serverplayback from mitmproxy import options from mitmproxy import proxy @@ -244,7 +244,7 @@ class TestServerPlayback: m.addons.add(s) f = tutils.tflow() - f.response = netlib.tutils.tresp(content=f.request.content) + f.response = mitmproxy.test.tutils.tresp(content=f.request.content) s.load([f, f]) tf = tutils.tflow() @@ -273,7 +273,7 @@ class TestServerPlayback: m.addons.add(s) f = tutils.tflow() - f.response = netlib.tutils.tresp(content=f.request.content) + f.response = mitmproxy.test.tutils.tresp(content=f.request.content) s.load([f]) f = tutils.tflow() diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index 29c9e198..28c35b28 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -3,7 +3,7 @@ from mitmproxy.addons import stickycookie from mitmproxy import master from mitmproxy import options from mitmproxy import proxy -from netlib import tutils as ntutils +from mitmproxy.test import tutils as ntutils def test_domain_match(): diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index da29a2b1..6d4bb7f9 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -1,6 +1,6 @@ import gc -import netlib.tutils +import mitmproxy.test.tutils from mitmproxy.tools import console from mitmproxy import proxy from mitmproxy.tools.console import common @@ -59,13 +59,13 @@ class TestConsoleState: def _add_response(self, state): f = self._add_request(state) - f.response = netlib.tutils.tresp() + f.response = mitmproxy.test.tutils.tresp() state.update_flow(f) def test_add_response(self): c = console.master.ConsoleState() f = self._add_request(c) - f.response = netlib.tutils.tresp() + f.response = mitmproxy.test.tutils.tresp() c.focus = None c.update_flow(f) @@ -127,12 +127,12 @@ class TestMaster(mastertest.MasterTest): def test_intercept(self): """regression test for https://github.com/mitmproxy/mitmproxy/issues/1605""" m = self.mkmaster(intercept="~b bar") - f = tutils.tflow(req=netlib.tutils.treq(content=b"foo")) + f = tutils.tflow(req=mitmproxy.test.tutils.treq(content=b"foo")) m.request(f) assert not m.state.flows[0].intercepted - f = tutils.tflow(req=netlib.tutils.treq(content=b"bar")) + f = tutils.tflow(req=mitmproxy.test.tutils.treq(content=b"bar")) m.request(f) assert m.state.flows[1].intercepted - f = tutils.tflow(resp=netlib.tutils.tresp(content=b"bar")) + f = tutils.tflow(resp=mitmproxy.test.tutils.tresp(content=b"bar")) m.request(f) assert m.state.flows[2].intercepted diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 0b57c2d4..cdfb5ad5 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -1,7 +1,7 @@ import contextlib from . import tutils -import netlib.tutils +import mitmproxy.test.tutils from mitmproxy import master from mitmproxy import io @@ -17,13 +17,13 @@ class TestMaster: class MasterTest: def cycle(self, master, content): - f = tutils.tflow(req=netlib.tutils.treq(content=content)) + f = tutils.tflow(req=mitmproxy.test.tutils.treq(content=content)) master.clientconnect(f.client_conn) master.serverconnect(f.server_conn) master.request(f) if not f.error: f.response = http.HTTPResponse.wrap( - netlib.tutils.tresp(content=content) + mitmproxy.test.tutils.tresp(content=content) ) master.response(f) master.clientdisconnect(f) diff --git a/test/netlib/http/http1/__init__.py b/test/mitmproxy/net/__init__.py index e69de29b..e69de29b 100644 --- a/test/netlib/http/http1/__init__.py +++ b/test/mitmproxy/net/__init__.py diff --git a/test/netlib/data/clientcert/.gitignore b/test/mitmproxy/net/data/clientcert/.gitignore index 07bc53d2..07bc53d2 100644 --- a/test/netlib/data/clientcert/.gitignore +++ b/test/mitmproxy/net/data/clientcert/.gitignore diff --git a/test/netlib/data/clientcert/client.cnf b/test/mitmproxy/net/data/clientcert/client.cnf index 5046a944..5046a944 100644 --- a/test/netlib/data/clientcert/client.cnf +++ b/test/mitmproxy/net/data/clientcert/client.cnf diff --git a/test/netlib/data/clientcert/client.pem b/test/mitmproxy/net/data/clientcert/client.pem index 4927bca2..4927bca2 100644 --- a/test/netlib/data/clientcert/client.pem +++ b/test/mitmproxy/net/data/clientcert/client.pem diff --git a/test/netlib/data/clientcert/make b/test/mitmproxy/net/data/clientcert/make index d1caea81..d1caea81 100644 --- a/test/netlib/data/clientcert/make +++ b/test/mitmproxy/net/data/clientcert/make diff --git a/test/netlib/data/dercert b/test/mitmproxy/net/data/dercert Binary files differindex 370252af..370252af 100644 --- a/test/netlib/data/dercert +++ b/test/mitmproxy/net/data/dercert diff --git a/test/netlib/data/dhparam.pem b/test/mitmproxy/net/data/dhparam.pem index afb41672..afb41672 100644 --- a/test/netlib/data/dhparam.pem +++ b/test/mitmproxy/net/data/dhparam.pem diff --git a/test/netlib/data/htpasswd b/test/mitmproxy/net/data/htpasswd index 54c95b8c..54c95b8c 100644 --- a/test/netlib/data/htpasswd +++ b/test/mitmproxy/net/data/htpasswd diff --git a/test/netlib/data/server.crt b/test/mitmproxy/net/data/server.crt index 68f61bac..68f61bac 100644 --- a/test/netlib/data/server.crt +++ b/test/mitmproxy/net/data/server.crt diff --git a/test/netlib/data/server.key b/test/mitmproxy/net/data/server.key index b1b658ab..b1b658ab 100644 --- a/test/netlib/data/server.key +++ b/test/mitmproxy/net/data/server.key diff --git a/test/netlib/data/text_cert b/test/mitmproxy/net/data/text_cert index 36ca33b9..36ca33b9 100644 --- a/test/netlib/data/text_cert +++ b/test/mitmproxy/net/data/text_cert diff --git a/test/netlib/data/text_cert_2 b/test/mitmproxy/net/data/text_cert_2 index ffe8faae..ffe8faae 100644 --- a/test/netlib/data/text_cert_2 +++ b/test/mitmproxy/net/data/text_cert_2 diff --git a/test/netlib/data/text_cert_weird1 b/test/mitmproxy/net/data/text_cert_weird1 index 72b09dcb..72b09dcb 100644 --- a/test/netlib/data/text_cert_weird1 +++ b/test/mitmproxy/net/data/text_cert_weird1 diff --git a/test/netlib/data/verificationcerts/9da13359.0 b/test/mitmproxy/net/data/verificationcerts/9da13359.0 index b22e4d20..b22e4d20 100644 --- a/test/netlib/data/verificationcerts/9da13359.0 +++ b/test/mitmproxy/net/data/verificationcerts/9da13359.0 diff --git a/test/netlib/data/verificationcerts/generate.py b/test/mitmproxy/net/data/verificationcerts/generate.py index 6d4d8550..6d4d8550 100644 --- a/test/netlib/data/verificationcerts/generate.py +++ b/test/mitmproxy/net/data/verificationcerts/generate.py diff --git a/test/netlib/data/verificationcerts/self-signed.crt b/test/mitmproxy/net/data/verificationcerts/self-signed.crt index dce2a7e0..dce2a7e0 100644 --- a/test/netlib/data/verificationcerts/self-signed.crt +++ b/test/mitmproxy/net/data/verificationcerts/self-signed.crt diff --git a/test/netlib/data/verificationcerts/self-signed.key b/test/mitmproxy/net/data/verificationcerts/self-signed.key index 71a6ad6a..71a6ad6a 100644 --- a/test/netlib/data/verificationcerts/self-signed.key +++ b/test/mitmproxy/net/data/verificationcerts/self-signed.key diff --git a/test/netlib/data/verificationcerts/trusted-leaf.crt b/test/mitmproxy/net/data/verificationcerts/trusted-leaf.crt index 6a92de92..6a92de92 100644 --- a/test/netlib/data/verificationcerts/trusted-leaf.crt +++ b/test/mitmproxy/net/data/verificationcerts/trusted-leaf.crt diff --git a/test/netlib/data/verificationcerts/trusted-leaf.key b/test/mitmproxy/net/data/verificationcerts/trusted-leaf.key index 783ebf1c..783ebf1c 100644 --- a/test/netlib/data/verificationcerts/trusted-leaf.key +++ b/test/mitmproxy/net/data/verificationcerts/trusted-leaf.key diff --git a/test/netlib/data/verificationcerts/trusted-root.crt b/test/mitmproxy/net/data/verificationcerts/trusted-root.crt index b22e4d20..b22e4d20 100644 --- a/test/netlib/data/verificationcerts/trusted-root.crt +++ b/test/mitmproxy/net/data/verificationcerts/trusted-root.crt diff --git a/test/netlib/data/verificationcerts/trusted-root.key b/test/mitmproxy/net/data/verificationcerts/trusted-root.key index 05483f77..05483f77 100644 --- a/test/netlib/data/verificationcerts/trusted-root.key +++ b/test/mitmproxy/net/data/verificationcerts/trusted-root.key diff --git a/test/netlib/data/verificationcerts/trusted-root.srl b/test/mitmproxy/net/data/verificationcerts/trusted-root.srl index 4ad962ba..4ad962ba 100644 --- a/test/netlib/data/verificationcerts/trusted-root.srl +++ b/test/mitmproxy/net/data/verificationcerts/trusted-root.srl diff --git a/test/netlib/http/http2/__init__.py b/test/mitmproxy/net/http/__init__.py index e69de29b..e69de29b 100644 --- a/test/netlib/http/http2/__init__.py +++ b/test/mitmproxy/net/http/__init__.py diff --git a/test/netlib/websockets/__init__.py b/test/mitmproxy/net/http/http1/__init__.py index e69de29b..e69de29b 100644 --- a/test/netlib/websockets/__init__.py +++ b/test/mitmproxy/net/http/http1/__init__.py diff --git a/test/netlib/http/http1/test_assemble.py b/test/mitmproxy/net/http/http1/test_assemble.py index 5d7e007e..e80376e8 100644 --- a/test/netlib/http/http1/test_assemble.py +++ b/test/mitmproxy/net/http/http1/test_assemble.py @@ -1,11 +1,11 @@ -from netlib.exceptions import HttpException -from netlib.http import Headers -from netlib.http.http1.assemble import ( +from mitmproxy import exceptions +from mitmproxy.net.http import Headers +from mitmproxy.net.http.http1.assemble import ( assemble_request, assemble_request_head, assemble_response, assemble_response_head, _assemble_request_line, _assemble_request_headers, _assemble_response_headers, assemble_body) -from netlib.tutils import treq, raises, tresp +from mitmproxy.test.tutils import treq, raises, tresp def test_assemble_request(): @@ -18,7 +18,7 @@ def test_assemble_request(): b"content" ) - with raises(HttpException): + with raises(exceptions.HttpException): assemble_request(treq(content=None)) @@ -39,7 +39,7 @@ def test_assemble_response(): b"message" ) - with raises(HttpException): + with raises(exceptions.HttpException): assemble_response(tresp(content=None)) diff --git a/test/netlib/http/http1/test_read.py b/test/mitmproxy/net/http/http1/test_read.py index f25cd3e2..20997259 100644 --- a/test/netlib/http/http1/test_read.py +++ b/test/mitmproxy/net/http/http1/test_read.py @@ -2,16 +2,15 @@ from io import BytesIO from mock import Mock import pytest -from netlib.exceptions import HttpException, HttpSyntaxException, HttpReadDisconnect, TcpDisconnect -from netlib.http import Headers -from netlib.http.http1.read import ( +from mitmproxy import exceptions +from mitmproxy.net.http import Headers +from mitmproxy.net.http.http1.read import ( read_request, read_response, read_request_head, read_response_head, read_body, connection_close, expected_http_body_size, _get_first_line, _read_request_line, _parse_authority_form, _read_response_line, _check_http_version, _read_headers, _read_chunked, get_header_tokens ) -from netlib.tutils import treq, tresp, raises -from netlib import exceptions +from mitmproxy.test.tutils import treq, tresp, raises def test_get_header_tokens(): @@ -117,12 +116,12 @@ class TestReadBody: def test_known_size_limit(self): rfile = BytesIO(b"foobar") - with raises(HttpException): + with raises(exceptions.HttpException): b"".join(read_body(rfile, 3, 2)) def test_known_size_too_short(self): rfile = BytesIO(b"foo") - with raises(HttpException): + with raises(exceptions.HttpException): b"".join(read_body(rfile, 6)) def test_unknown_size(self): @@ -132,7 +131,7 @@ class TestReadBody: def test_unknown_size_limit(self): rfile = BytesIO(b"foobar") - with raises(HttpException): + with raises(exceptions.HttpException): b"".join(read_body(rfile, -1, 3)) def test_max_chunk_size(self): @@ -186,7 +185,7 @@ def test_expected_http_body_size(): # explicit length for val in (b"foo", b"-7"): - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): expected_http_body_size( treq(headers=Headers(content_length=val)) ) @@ -210,13 +209,13 @@ def test_get_first_line(): rfile = BytesIO(b"\r\nfoo\r\nbar") assert _get_first_line(rfile) == b"foo" - with raises(HttpReadDisconnect): + with raises(exceptions.HttpReadDisconnect): rfile = BytesIO(b"") _get_first_line(rfile) - with raises(HttpReadDisconnect): + with raises(exceptions.HttpReadDisconnect): rfile = Mock() - rfile.readline.side_effect = TcpDisconnect + rfile.readline.side_effect = exceptions.TcpDisconnect _get_first_line(rfile) @@ -233,23 +232,23 @@ def test_read_request_line(): assert (t(b"GET http://foo:42/bar HTTP/1.1") == ("absolute", b"GET", b"http", b"foo", 42, b"/bar", b"HTTP/1.1")) - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): t(b"GET / WTF/1.1") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): t(b"this is not http") - with raises(HttpReadDisconnect): + with raises(exceptions.HttpReadDisconnect): t(b"") def test_parse_authority_form(): assert _parse_authority_form(b"foo:42") == (b"foo", 42) - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _parse_authority_form(b"foo") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _parse_authority_form(b"foo:bar") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _parse_authority_form(b"foo:99999999") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _parse_authority_form(b"f\x00oo:80") @@ -263,14 +262,14 @@ def test_read_response_line(): # https://github.com/mitmproxy/mitmproxy/issues/784 assert t(b"HTTP/1.1 200 Non-Autoris\xc3\xa9") == (b"HTTP/1.1", 200, b"Non-Autoris\xc3\xa9") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): assert t(b"HTTP/1.1") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): t(b"HTTP/1.1 OK OK") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): t(b"WTF/1.1 200 OK") - with raises(HttpReadDisconnect): + with raises(exceptions.HttpReadDisconnect): t(b"") @@ -279,11 +278,11 @@ def test_check_http_version(): _check_http_version(b"HTTP/1.0") _check_http_version(b"HTTP/1.1") _check_http_version(b"HTTP/2.0") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _check_http_version(b"WTF/1.0") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _check_http_version(b"HTTP/1.10") - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): _check_http_version(b"HTTP/1.b") @@ -322,17 +321,17 @@ class TestReadHeaders: def test_read_continued_err(self): data = b"\tfoo: bar\r\n" - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): self._read(data) def test_read_err(self): data = b"foo" - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): self._read(data) def test_read_empty_name(self): data = b":foo" - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): self._read(data) def test_read_empty_value(self): @@ -346,7 +345,7 @@ def test_read_chunked(): req.headers["Transfer-Encoding"] = "chunked" data = b"1\r\na\r\n0\r\n" - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): b"".join(_read_chunked(BytesIO(data))) data = b"1\r\na\r\n0\r\n\r\n" @@ -364,7 +363,7 @@ def test_read_chunked(): b"".join(_read_chunked(BytesIO(data))) data = b"foo\r\nfoo" - with raises(HttpSyntaxException): + with raises(exceptions.HttpSyntaxException): b"".join(_read_chunked(BytesIO(data))) data = b"5\r\naaaaa\r\n0\r\n\r\n" diff --git a/test/mitmproxy/net/http/http2/__init__.py b/test/mitmproxy/net/http/http2/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/net/http/http2/__init__.py diff --git a/test/netlib/http/http2/test_framereader.py b/test/mitmproxy/net/http/http2/test_framereader.py index 41b73189..41b73189 100644 --- a/test/netlib/http/http2/test_framereader.py +++ b/test/mitmproxy/net/http/http2/test_framereader.py diff --git a/test/netlib/http/test_authentication.py b/test/mitmproxy/net/http/test_authentication.py index 95d72447..11b6e2e0 100644 --- a/test/netlib/http/test_authentication.py +++ b/test/mitmproxy/net/http/test_authentication.py @@ -1,7 +1,7 @@ import binascii -from netlib import tutils -from netlib.http import authentication, Headers +from mitmproxy.test import tutils +from mitmproxy.net.http import authentication, Headers def test_parse_http_basic_auth(): diff --git a/test/netlib/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 80ffb0a6..8c9c0c32 100644 --- a/test/netlib/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -1,7 +1,7 @@ import time -from netlib.http import cookies -from netlib.tutils import raises +from mitmproxy.net.http import cookies +from mitmproxy.test.tutils import raises import mock diff --git a/test/netlib/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py index e1175ef0..d8fa5e76 100644 --- a/test/netlib/test_encoding.py +++ b/test/mitmproxy/net/http/test_encoding.py @@ -1,7 +1,8 @@ import mock import pytest -from netlib import encoding, tutils +from mitmproxy.net.http import encoding +from mitmproxy.test import tutils @pytest.mark.parametrize("encoder", [ diff --git a/test/netlib/http/test_headers.py b/test/mitmproxy/net/http/test_headers.py index 63f16897..8e0f770d 100644 --- a/test/netlib/http/test_headers.py +++ b/test/mitmproxy/net/http/test_headers.py @@ -1,7 +1,7 @@ import collections -from netlib.http.headers import Headers, parse_content_type, assemble_content_type -from netlib.tutils import raises +from mitmproxy.net.http.headers import Headers, parse_content_type, assemble_content_type +from mitmproxy.test.tutils import raises class TestHeaders: diff --git a/test/netlib/http/test_message.py b/test/mitmproxy/net/http/test_message.py index 8374f8f6..69d029d9 100644 --- a/test/netlib/http/test_message.py +++ b/test/mitmproxy/net/http/test_message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from netlib.tutils import tresp -from netlib import http, tutils +from mitmproxy.test import tutils +from mitmproxy.net import http def _test_passthrough_attr(message, attr): @@ -37,12 +37,12 @@ def _test_decoded_attr(message, attr): class TestMessageData: def test_eq_ne(self): - data = tresp(timestamp_start=42, timestamp_end=42).data - same = tresp(timestamp_start=42, timestamp_end=42).data + data = tutils.tresp(timestamp_start=42, timestamp_end=42).data + same = tutils.tresp(timestamp_start=42, timestamp_end=42).data assert data == same assert not data != same - other = tresp(content=b"foo").data + other = tutils.tresp(content=b"foo").data assert not data == other assert data != other @@ -52,28 +52,28 @@ class TestMessageData: class TestMessage: def test_init(self): - resp = tresp() + resp = tutils.tresp() assert resp.data def test_eq_ne(self): - resp = tresp(timestamp_start=42, timestamp_end=42) - same = tresp(timestamp_start=42, timestamp_end=42) + resp = tutils.tresp(timestamp_start=42, timestamp_end=42) + same = tutils.tresp(timestamp_start=42, timestamp_end=42) assert resp == same assert not resp != same - other = tresp(timestamp_start=0, timestamp_end=0) + other = tutils.tresp(timestamp_start=0, timestamp_end=0) assert not resp == other assert resp != other assert resp != 0 def test_serializable(self): - resp = tresp() + resp = tutils.tresp() resp2 = http.Response.from_state(resp.get_state()) assert resp == resp2 def test_content_length_update(self): - resp = tresp() + resp = tutils.tresp() resp.content = b"foo" assert resp.data.content == b"foo" assert resp.headers["content-length"] == "3" @@ -85,19 +85,19 @@ class TestMessage: assert resp.headers["content-length"] == "0" def test_headers(self): - _test_passthrough_attr(tresp(), "headers") + _test_passthrough_attr(tutils.tresp(), "headers") def test_timestamp_start(self): - _test_passthrough_attr(tresp(), "timestamp_start") + _test_passthrough_attr(tutils.tresp(), "timestamp_start") def test_timestamp_end(self): - _test_passthrough_attr(tresp(), "timestamp_end") + _test_passthrough_attr(tutils.tresp(), "timestamp_end") def test_http_version(self): - _test_decoded_attr(tresp(), "http_version") + _test_decoded_attr(tutils.tresp(), "http_version") def test_replace(self): - r = tresp() + r = tutils.tresp() r.content = b"foofootoo" r.replace(b"foo", "gg") assert r.content == b"ggggtoo" @@ -109,7 +109,7 @@ class TestMessage: class TestMessageContentEncoding: def test_simple(self): - r = tresp() + r = tutils.tresp() assert r.raw_content == b"message" assert "content-encoding" not in r.headers r.encode("gzip") @@ -120,7 +120,7 @@ class TestMessageContentEncoding: assert r.raw_content != b"message" def test_modify(self): - r = tresp() + r = tutils.tresp() assert "content-encoding" not in r.headers r.encode("gzip") @@ -133,7 +133,7 @@ class TestMessageContentEncoding: r.content = u"foo" def test_unknown_ce(self): - r = tresp() + r = tutils.tresp() r.headers["content-encoding"] = "zopfli" r.raw_content = b"foo" with tutils.raises(ValueError): @@ -142,7 +142,7 @@ class TestMessageContentEncoding: assert r.get_content(strict=False) == b"foo" def test_cannot_decode(self): - r = tresp() + r = tutils.tresp() r.encode("gzip") r.raw_content = b"foo" with tutils.raises(ValueError): @@ -160,7 +160,7 @@ class TestMessageContentEncoding: assert "content-encoding" not in r.headers def test_none(self): - r = tresp(content=None) + r = tutils.tresp(content=None) assert r.content is None r.content = b"foo" assert r.content is not None @@ -168,7 +168,7 @@ class TestMessageContentEncoding: assert r.content is None def test_cannot_encode(self): - r = tresp() + r = tutils.tresp() r.encode("gzip") r.content = None assert r.headers["content-encoding"] @@ -187,7 +187,7 @@ class TestMessageContentEncoding: class TestMessageText: def test_simple(self): - r = tresp(content=b'\xfc') + r = tutils.tresp(content=b'\xfc') assert r.raw_content == b"\xfc" assert r.content == b"\xfc" assert r.text == u"ü" @@ -204,12 +204,12 @@ class TestMessageText: assert r.text == u"ü" def test_guess_json(self): - r = tresp(content=b'"\xc3\xbc"') + r = tutils.tresp(content=b'"\xc3\xbc"') r.headers["content-type"] = "application/json" assert r.text == u'"ü"' def test_none(self): - r = tresp(content=None) + r = tutils.tresp(content=None) assert r.text is None r.text = u"foo" assert r.text is not None @@ -217,7 +217,7 @@ class TestMessageText: assert r.text is None def test_modify(self): - r = tresp() + r = tutils.tresp() r.text = u"ü" assert r.raw_content == b"\xfc" @@ -228,7 +228,7 @@ class TestMessageText: assert r.headers["content-length"] == "2" def test_unknown_ce(self): - r = tresp() + r = tutils.tresp() r.headers["content-type"] = "text/html; charset=wtf" r.raw_content = b"foo" with tutils.raises(ValueError): @@ -236,7 +236,7 @@ class TestMessageText: assert r.get_text(strict=False) == u"foo" def test_cannot_decode(self): - r = tresp() + r = tutils.tresp() r.headers["content-type"] = "text/html; charset=utf8" r.raw_content = b"\xFF" with tutils.raises(ValueError): @@ -245,7 +245,7 @@ class TestMessageText: assert r.get_text(strict=False) == '\udcff' def test_cannot_encode(self): - r = tresp() + r = tutils.tresp() r.content = None assert "content-type" not in r.headers assert r.raw_content is None diff --git a/test/netlib/http/test_multipart.py b/test/mitmproxy/net/http/test_multipart.py index 1d7e0062..68ae6bbd 100644 --- a/test/netlib/http/test_multipart.py +++ b/test/mitmproxy/net/http/test_multipart.py @@ -1,5 +1,5 @@ -from netlib.http import Headers -from netlib.http import multipart +from mitmproxy.net.http import Headers +from mitmproxy.net.http import multipart def test_decode(): diff --git a/test/netlib/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 336dc86d..9c0ec333 100644 --- a/test/netlib/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from netlib.http import Headers -from netlib.tutils import treq, raises +from mitmproxy.net.http import Headers +from mitmproxy.test.tutils import treq, raises from .test_message import _test_decoded_attr, _test_passthrough_attr diff --git a/test/netlib/http/test_response.py b/test/mitmproxy/net/http/test_response.py index 725f2b33..239fb6ef 100644 --- a/test/netlib/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -2,10 +2,10 @@ import email import time -from netlib.http import Headers -from netlib.http import Response -from netlib.http.cookies import CookieAttrs -from netlib.tutils import raises, tresp +from mitmproxy.net.http import Headers +from mitmproxy.net.http import Response +from mitmproxy.net.http.cookies import CookieAttrs +from mitmproxy.test.tutils import raises, tresp from .test_message import _test_passthrough_attr, _test_decoded_attr diff --git a/test/netlib/http/test_status_codes.py b/test/mitmproxy/net/http/test_status_codes.py index 9fea6b70..098f8eef 100644 --- a/test/netlib/http/test_status_codes.py +++ b/test/mitmproxy/net/http/test_status_codes.py @@ -1,4 +1,4 @@ -from netlib.http import status_codes +from mitmproxy.net.http import status_codes def test_simple(): diff --git a/test/netlib/http/test_url.py b/test/mitmproxy/net/http/test_url.py index 631ed8a9..94b2eacc 100644 --- a/test/netlib/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -1,5 +1,5 @@ -from netlib import tutils -from netlib.http import url +from mitmproxy.test import tutils +from mitmproxy.net.http import url def test_parse(): diff --git a/test/netlib/http/test_user_agents.py b/test/mitmproxy/net/http/test_user_agents.py index 0bf1bba7..9e391447 100644 --- a/test/netlib/http/test_user_agents.py +++ b/test/mitmproxy/net/http/test_user_agents.py @@ -1,4 +1,4 @@ -from netlib.http import user_agents +from mitmproxy.net.http import user_agents def test_get_shortcut(): diff --git a/test/mitmproxy/net/test_check.py b/test/mitmproxy/net/test_check.py new file mode 100644 index 00000000..18a50157 --- /dev/null +++ b/test/mitmproxy/net/test_check.py @@ -0,0 +1,10 @@ +# coding=utf-8 + +from mitmproxy.net import check + + +def test_is_valid_host(): + assert not check.is_valid_host(b"") + assert check.is_valid_host(b"one.two") + assert not check.is_valid_host(b"one" * 255) + assert check.is_valid_host(b"one.two.") diff --git a/test/netlib/test_imports.py b/test/mitmproxy/net/test_imports.py index b88ef26d..b88ef26d 100644 --- a/test/netlib/test_imports.py +++ b/test/mitmproxy/net/test_imports.py diff --git a/test/netlib/test_socks.py b/test/mitmproxy/net/test_socks.py index 17e08054..b6551faa 100644 --- a/test/netlib/test_socks.py +++ b/test/mitmproxy/net/test_socks.py @@ -1,6 +1,8 @@ import ipaddress from io import BytesIO -from netlib import socks, tcp, tutils +from mitmproxy.net import socks +from mitmproxy.net import tcp +from mitmproxy.test import tutils def test_client_greeting(): diff --git a/test/netlib/test_tcp.py b/test/mitmproxy/net/test_tcp.py index 797a5a04..8a7c692a 100644 --- a/test/netlib/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -9,9 +9,10 @@ import mock from OpenSSL import SSL -from netlib import tcp, certutils, tutils -from netlib.exceptions import InvalidCertificateException, TcpReadIncomplete, TlsException, \ - TcpTimeout, TcpDisconnect, TcpException, NetlibException +from mitmproxy import certs +from mitmproxy.net import tcp +from mitmproxy.test import tutils +from mitmproxy import exceptions from . import tservers @@ -108,7 +109,7 @@ class TestServerBind(tservers.ServerTestBase): with c.connect(): assert c.rfile.readline() == str(("127.0.0.1", random_port)).encode() return - except TcpException: # port probably already in use + except exceptions.TcpException: # port probably already in use pass @@ -155,7 +156,7 @@ class TestFinishFail(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.wfile.write(b"foo\n") - c.wfile.flush = mock.Mock(side_effect=TcpDisconnect) + c.wfile.flush = mock.Mock(side_effect=exceptions.TcpDisconnect) c.finish() @@ -195,7 +196,7 @@ class TestSSLv3Only(tservers.ServerTestBase): def test_failure(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - tutils.raises(TlsException, c.convert_to_ssl, sni="foo.com") + tutils.raises(exceptions.TlsException, c.convert_to_ssl, sni="foo.com") class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): @@ -236,7 +237,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): def test_mode_strict_should_fail(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - with tutils.raises(InvalidCertificateException): + with tutils.raises(exceptions.InvalidCertificateException): c.convert_to_ssl( sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, @@ -261,7 +262,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): def test_should_fail_without_sni(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - with tutils.raises(TlsException): + with tutils.raises(exceptions.TlsException): c.convert_to_ssl( verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") @@ -270,7 +271,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): def test_should_fail(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - with tutils.raises(InvalidCertificateException): + with tutils.raises(exceptions.InvalidCertificateException): c.convert_to_ssl( sni="mitmproxy.org", verify_options=SSL.VERIFY_PEER, @@ -348,7 +349,7 @@ class TestSSLClientCert(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): tutils.raises( - TlsException, + exceptions.TlsException, c.convert_to_ssl, cert=tutils.test_data.path("data/clientcert/make") ) @@ -454,7 +455,7 @@ class TestSSLDisconnect(tservers.ServerTestBase): # Excercise SSL.ZeroReturnError c.rfile.read(10) c.close() - tutils.raises(TcpDisconnect, c.wfile.write, b"foo") + tutils.raises(exceptions.TcpDisconnect, c.wfile.write, b"foo") tutils.raises(queue.Empty, self.q.get_nowait) @@ -469,7 +470,7 @@ class TestSSLHardDisconnect(tservers.ServerTestBase): # Exercise SSL.SysCallError c.rfile.read(10) c.close() - tutils.raises(TcpDisconnect, c.wfile.write, b"foo") + tutils.raises(exceptions.TcpDisconnect, c.wfile.write, b"foo") class TestDisconnect(tservers.ServerTestBase): @@ -492,7 +493,7 @@ class TestServerTimeOut(tservers.ServerTestBase): self.settimeout(0.01) try: self.rfile.read(10) - except TcpTimeout: + except exceptions.TcpTimeout: self.timeout = True def test_timeout(self): @@ -510,7 +511,7 @@ class TestTimeOut(tservers.ServerTestBase): with c.connect(): c.settimeout(0.1) assert c.gettimeout() == 0.1 - tutils.raises(TcpTimeout, c.rfile.read, 10) + tutils.raises(exceptions.TcpTimeout, c.rfile.read, 10) class TestALPNClient(tservers.ServerTestBase): @@ -562,13 +563,13 @@ class TestSSLTimeOut(tservers.ServerTestBase): with c.connect(): c.convert_to_ssl() c.settimeout(0.1) - tutils.raises(TcpTimeout, c.rfile.read, 10) + tutils.raises(exceptions.TcpTimeout, c.rfile.read, 10) class TestDHParams(tservers.ServerTestBase): handler = HangHandler ssl = dict( - dhparams=certutils.CertStore.load_dhparam( + dhparams=certs.CertStore.load_dhparam( tutils.test_data.path("data/dhparam.pem"), ), cipher_list="DHE-RSA-AES256-SHA" @@ -584,7 +585,7 @@ class TestDHParams(tservers.ServerTestBase): def test_create_dhparams(self): with tutils.tmpdir() as d: filename = os.path.join(d, "dhparam.pem") - certutils.CertStore.load_dhparam(filename) + certs.CertStore.load_dhparam(filename) assert os.path.exists(filename) @@ -592,7 +593,7 @@ class TestTCPClient: def test_conerr(self): c = tcp.TCPClient(("127.0.0.1", 0)) - tutils.raises(TcpException, c.connect) + tutils.raises(exceptions.TcpException, c.connect) class TestFileLike: @@ -661,7 +662,7 @@ class TestFileLike: o = mock.MagicMock() o.flush = mock.MagicMock(side_effect=socket.error) s.o = o - tutils.raises(TcpDisconnect, s.flush) + tutils.raises(exceptions.TcpDisconnect, s.flush) def test_reader_read_error(self): s = BytesIO(b"foobar\nfoobar") @@ -669,7 +670,7 @@ class TestFileLike: o = mock.MagicMock() o.read = mock.MagicMock(side_effect=socket.error) s.o = o - tutils.raises(TcpDisconnect, s.read, 10) + tutils.raises(exceptions.TcpDisconnect, s.read, 10) def test_reset_timestamps(self): s = BytesIO(b"foobar\nfoobar") @@ -700,24 +701,24 @@ class TestFileLike: s = mock.MagicMock() s.read = mock.MagicMock(side_effect=SSL.Error()) s = tcp.Reader(s) - tutils.raises(TlsException, s.read, 1) + tutils.raises(exceptions.TlsException, s.read, 1) def test_read_syscall_ssl_error(self): s = mock.MagicMock() s.read = mock.MagicMock(side_effect=SSL.SysCallError()) s = tcp.Reader(s) - tutils.raises(TlsException, s.read, 1) + tutils.raises(exceptions.TlsException, s.read, 1) def test_reader_readline_disconnect(self): o = mock.MagicMock() o.read = mock.MagicMock(side_effect=socket.error) s = tcp.Reader(o) - tutils.raises(TcpDisconnect, s.readline, 10) + tutils.raises(exceptions.TcpDisconnect, s.readline, 10) def test_reader_incomplete_error(self): s = BytesIO(b"foobar") s = tcp.Reader(s) - tutils.raises(TcpReadIncomplete, s.safe_read, 10) + tutils.raises(exceptions.TcpReadIncomplete, s.safe_read, 10) class TestPeek(tservers.ServerTestBase): @@ -738,11 +739,11 @@ class TestPeek(tservers.ServerTestBase): assert c.rfile.readline() == testval c.close() - with tutils.raises(NetlibException): + with tutils.raises(exceptions.NetlibException): if c.rfile.peek(1) == b"": # Workaround for Python 2 on Unix: # Peeking a closed connection does not raise an exception here. - raise NetlibException() + raise exceptions.NetlibException() class TestPeekSSL(TestPeek): diff --git a/test/netlib/test_wsgi.py b/test/mitmproxy/net/test_wsgi.py index 5c61f81c..b4d6b53f 100644 --- a/test/netlib/test_wsgi.py +++ b/test/mitmproxy/net/test_wsgi.py @@ -1,7 +1,7 @@ from io import BytesIO import sys -from netlib import wsgi -from netlib.http import Headers +from mitmproxy.net import wsgi +from mitmproxy.net.http import Headers def tflow(): diff --git a/test/netlib/tools/getcertnames b/test/mitmproxy/net/tools/getcertnames index 0882fccd..d64e5ff5 100644 --- a/test/netlib/tools/getcertnames +++ b/test/mitmproxy/net/tools/getcertnames @@ -1,7 +1,7 @@ #!/usr/bin/env python import sys sys.path.insert(0, "../../") -from netlib import tcp +from mitmproxy.net import tcp def get_remote_cert(host, port, sni): diff --git a/test/netlib/tservers.py b/test/mitmproxy/net/tservers.py index b344e25c..ba10ef5e 100644 --- a/test/netlib/tservers.py +++ b/test/mitmproxy/net/tservers.py @@ -3,8 +3,8 @@ import queue import io import OpenSSL -from netlib import tcp -from netlib import tutils +from mitmproxy.net import tcp +from mitmproxy.test import tutils class _ServerThread(threading.Thread): diff --git a/test/mitmproxy/net/websockets/__init__.py b/test/mitmproxy/net/websockets/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/net/websockets/__init__.py diff --git a/test/netlib/websockets/test_frame.py b/test/mitmproxy/net/websockets/test_frame.py index a039dcb1..198be3d5 100644 --- a/test/netlib/websockets/test_frame.py +++ b/test/mitmproxy/net/websockets/test_frame.py @@ -2,8 +2,8 @@ import os import codecs import pytest -from netlib import websockets -from netlib import tutils +from mitmproxy.net import websockets +from mitmproxy.test import tutils class TestFrameHeader: diff --git a/test/netlib/websockets/test_masker.py b/test/mitmproxy/net/websockets/test_masker.py index 23a9b876..88360c3b 100644 --- a/test/netlib/websockets/test_masker.py +++ b/test/mitmproxy/net/websockets/test_masker.py @@ -1,7 +1,7 @@ import codecs import pytest -from netlib import websockets +from mitmproxy.net import websockets class TestMasker: diff --git a/test/netlib/websockets/test_utils.py b/test/mitmproxy/net/websockets/test_utils.py index f6f1e40a..6704d39d 100644 --- a/test/netlib/websockets/test_utils.py +++ b/test/mitmproxy/net/websockets/test_utils.py @@ -1,7 +1,7 @@ import pytest -from netlib import http -from netlib import websockets +from mitmproxy.net import http +from mitmproxy.net import websockets class TestUtils: diff --git a/test/mitmproxy/protocol/test_http1.py b/test/mitmproxy/protocol/test_http1.py index 8701c8e6..43c903f3 100644 --- a/test/mitmproxy/protocol/test_http1.py +++ b/test/mitmproxy/protocol/test_http1.py @@ -1,6 +1,6 @@ -from netlib.http import http1 -from netlib.tcp import TCPClient -from netlib.tutils import treq +from mitmproxy.net.http import http1 +from mitmproxy.net.tcp import TCPClient +from mitmproxy.test.tutils import treq from .. import tutils, tservers diff --git a/test/mitmproxy/protocol/test_http2.py b/test/mitmproxy/protocol/test_http2.py index a2efdc47..4629d109 100644 --- a/test/mitmproxy/protocol/test_http2.py +++ b/test/mitmproxy/protocol/test_http2.py @@ -11,10 +11,10 @@ import h2 from mitmproxy import options from mitmproxy.proxy.config import ProxyConfig -import netlib -from ...netlib import tservers as netlib_tservers -from netlib.exceptions import HttpException -from netlib.http import http1, http2 +import mitmproxy.net +from ...mitmproxy.net import tservers as net_tservers +from mitmproxy import exceptions +from mitmproxy.net.http import http1, http2 from .. import tservers @@ -28,7 +28,7 @@ logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) requires_alpn = pytest.mark.skipif( - not netlib.tcp.HAS_ALPN, + not mitmproxy.net.tcp.HAS_ALPN, reason='requires OpenSSL with ALPN support') @@ -37,10 +37,10 @@ requires_alpn = pytest.mark.skipif( # print(msg) -class _Http2ServerBase(netlib_tservers.ServerTestBase): +class _Http2ServerBase(net_tservers.ServerTestBase): ssl = dict(alpn_select=b'h2') - class handler(netlib.tcp.BaseHandler): + class handler(mitmproxy.net.tcp.BaseHandler): def handle(self): h2_conn = h2.connection.H2Connection(client_side=False, header_encoding=False) @@ -61,10 +61,10 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): try: raw = b''.join(http2.read_raw_frame(self.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False - except netlib.exceptions.TcpDisconnect: + except exceptions.TcpDisconnect: break except: print(traceback.format_exc()) @@ -77,7 +77,7 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): done = True break - except netlib.exceptions.TcpDisconnect: + except exceptions.TcpDisconnect: done = True except: done = True @@ -122,11 +122,11 @@ class _Http2TestBase: self.server.server.handle_server_event = self.handle_server_event def _setup_connection(self): - client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port)) + client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port)) client.connect() # send CONNECT request - client.wfile.write(http1.assemble_request(netlib.http.Request( + client.wfile.write(http1.assemble_request(mitmproxy.net.http.Request( 'authority', b'CONNECT', b'', @@ -252,7 +252,7 @@ class TestSimple(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -329,7 +329,7 @@ class TestRequestWithPriority(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -368,7 +368,7 @@ class TestRequestWithPriority(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -440,7 +440,7 @@ class TestPriority(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -517,7 +517,7 @@ class TestPriorityWithExistingStream(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -568,7 +568,7 @@ class TestStreamResetFromServer(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -618,7 +618,7 @@ class TestBodySizeLimit(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -703,7 +703,7 @@ class TestPushPromise(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False except: @@ -756,7 +756,7 @@ class TestPushPromise(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) events = h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False @@ -816,7 +816,7 @@ class TestConnectionLost(_Http2Test): try: raw = b''.join(http2.read_raw_frame(client.rfile)) h2_conn.receive_data(raw) - except HttpException: + except exceptions.HttpException: print(traceback.format_exc()) assert False except: diff --git a/test/mitmproxy/protocol/test_websockets.py b/test/mitmproxy/protocol/test_websockets.py index ef520d87..71cbb5f4 100644 --- a/test/mitmproxy/protocol/test_websockets.py +++ b/test/mitmproxy/protocol/test_websockets.py @@ -4,19 +4,20 @@ import tempfile import traceback from mitmproxy import options +from mitmproxy import exceptions from mitmproxy.proxy.config import ProxyConfig -import netlib -from netlib import http -from ...netlib import tservers as netlib_tservers +import mitmproxy.net +from mitmproxy.net import http +from ...mitmproxy.net import tservers as net_tservers from .. import tservers -from netlib import websockets +from mitmproxy.net import websockets -class _WebSocketsServerBase(netlib_tservers.ServerTestBase): +class _WebSocketsServerBase(net_tservers.ServerTestBase): - class handler(netlib.tcp.BaseHandler): + class handler(mitmproxy.net.tcp.BaseHandler): def handle(self): try: @@ -77,7 +78,7 @@ class _WebSocketsTestBase: self.server.server.handle_websockets = self.handle_websockets def _setup_connection(self): - client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port)) + client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port)) client.connect() request = http.Request( @@ -249,7 +250,7 @@ class TestClose(_WebSocketsTest): wfile.write(bytes(frame)) wfile.flush() - with pytest.raises(netlib.exceptions.TcpDisconnect): + with pytest.raises(exceptions.TcpDisconnect): websockets.Frame.from_file(rfile) def test_close(self): @@ -258,7 +259,7 @@ class TestClose(_WebSocketsTest): client.wfile.write(bytes(websockets.Frame(fin=1, opcode=websockets.OPCODE.CLOSE))) client.wfile.flush() - with pytest.raises(netlib.exceptions.TcpDisconnect): + with pytest.raises(exceptions.TcpDisconnect): websockets.Frame.from_file(client.rfile) def test_close_payload_1(self): @@ -267,7 +268,7 @@ class TestClose(_WebSocketsTest): client.wfile.write(bytes(websockets.Frame(fin=1, opcode=websockets.OPCODE.CLOSE, payload=b'\00\42'))) client.wfile.flush() - with pytest.raises(netlib.exceptions.TcpDisconnect): + with pytest.raises(exceptions.TcpDisconnect): websockets.Frame.from_file(client.rfile) def test_close_payload_2(self): @@ -276,7 +277,7 @@ class TestClose(_WebSocketsTest): client.wfile.write(bytes(websockets.Frame(fin=1, opcode=websockets.OPCODE.CLOSE, payload=b'\00\42foobar'))) client.wfile.flush() - with pytest.raises(netlib.exceptions.TcpDisconnect): + with pytest.raises(exceptions.TcpDisconnect): websockets.Frame.from_file(client.rfile) @@ -290,7 +291,7 @@ class TestInvalidFrame(_WebSocketsTest): def test_invalid_frame(self): client = self._setup_connection() - # with pytest.raises(netlib.exceptions.TcpDisconnect): + # with pytest.raises(exceptions.TcpDisconnect): frame = websockets.Frame.from_file(client.rfile) assert frame.header.opcode == 15 assert frame.payload == b'foobar' diff --git a/test/netlib/test_certutils.py b/test/mitmproxy/test_certs.py index cf9a671b..2e706fa6 100644 --- a/test/netlib/test_certutils.py +++ b/test/mitmproxy/test_certs.py @@ -1,9 +1,10 @@ import os -from netlib import certutils, tutils +from mitmproxy import certs +from mitmproxy.test import tutils # class TestDNTree: # def test_simple(self): -# d = certutils.DNTree() +# d = certs.DNTree() # d.add("foo.com", "foo") # d.add("bar.com", "bar") # assert d.get("foo.com") == "foo" @@ -19,12 +20,12 @@ from netlib import certutils, tutils # assert d.get("foo.foo.match.org") == "match" # # def test_wildcard(self): -# d = certutils.DNTree() +# d = certs.DNTree() # d.add("foo.com", "foo") # assert not d.get("*.foo.com") # d.add("*.foo.com", "wild") # -# d = certutils.DNTree() +# d = certs.DNTree() # d.add("*", "foo") # assert d.get("foo.com") == "foo" # assert d.get("*.foo.com") == "foo" @@ -35,22 +36,22 @@ class TestCertStore: def test_create_explicit(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") + ca = certs.CertStore.from_store(d, "test") assert ca.get_cert(b"foo", []) - ca2 = certutils.CertStore.from_store(d, "test") + ca2 = certs.CertStore.from_store(d, "test") assert ca2.get_cert(b"foo", []) assert ca.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() def test_create_no_common_name(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") + ca = certs.CertStore.from_store(d, "test") assert ca.get_cert(None, [])[0].cn is None def test_create_tmp(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") + ca = certs.CertStore.from_store(d, "test") assert ca.get_cert(b"foo.com", []) assert ca.get_cert(b"foo.com", []) assert ca.get_cert(b"*.foo.com", []) @@ -60,7 +61,7 @@ class TestCertStore: def test_sans(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") + ca = certs.CertStore.from_store(d, "test") c1 = ca.get_cert(b"foo.com", [b"*.bar.com"]) ca.get_cert(b"foo.bar.com", []) # assert c1 == c2 @@ -69,14 +70,14 @@ class TestCertStore: def test_sans_change(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") + ca = certs.CertStore.from_store(d, "test") ca.get_cert(b"foo.com", [b"*.bar.com"]) cert, key, chain_file = ca.get_cert(b"foo.bar.com", [b"*.baz.com"]) assert b"*.baz.com" in cert.altnames def test_expire(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") + ca = certs.CertStore.from_store(d, "test") ca.STORE_CAP = 3 ca.get_cert(b"one.com", []) ca.get_cert(b"two.com", []) @@ -101,8 +102,8 @@ class TestCertStore: def test_overrides(self): with tutils.tmpdir() as d: - ca1 = certutils.CertStore.from_store(os.path.join(d, "ca1"), "test") - ca2 = certutils.CertStore.from_store(os.path.join(d, "ca2"), "test") + ca1 = certs.CertStore.from_store(os.path.join(d, "ca1"), "test") + ca2 = certs.CertStore.from_store(os.path.join(d, "ca2"), "test") assert not ca1.default_ca.get_serial_number( ) == ca2.default_ca.get_serial_number() @@ -121,8 +122,8 @@ class TestDummyCert: def test_with_ca(self): with tutils.tmpdir() as d: - ca = certutils.CertStore.from_store(d, "test") - r = certutils.dummy_cert( + ca = certs.CertStore.from_store(d, "test") + r = certs.dummy_cert( ca.default_privatekey, ca.default_ca, b"foo.com", @@ -130,7 +131,7 @@ class TestDummyCert: ) assert r.cn == b"foo.com" - r = certutils.dummy_cert( + r = certs.dummy_cert( ca.default_privatekey, ca.default_ca, None, @@ -144,13 +145,13 @@ class TestSSLCert: def test_simple(self): with open(tutils.test_data.path("data/text_cert"), "rb") as f: d = f.read() - c1 = certutils.SSLCert.from_pem(d) + c1 = certs.SSLCert.from_pem(d) assert c1.cn == b"google.com" assert len(c1.altnames) == 436 with open(tutils.test_data.path("data/text_cert_2"), "rb") as f: d = f.read() - c2 = certutils.SSLCert.from_pem(d) + c2 = certs.SSLCert.from_pem(d) assert c2.cn == b"www.inode.co.nz" assert len(c2.altnames) == 2 assert c2.digest("sha1") @@ -169,12 +170,12 @@ class TestSSLCert: def test_err_broken_sans(self): with open(tutils.test_data.path("data/text_cert_weird1"), "rb") as f: d = f.read() - c = certutils.SSLCert.from_pem(d) + c = certs.SSLCert.from_pem(d) # This breaks unless we ignore a decoding error. assert c.altnames is not None def test_der(self): with open(tutils.test_data.path("data/dercert"), "rb") as f: d = f.read() - s = certutils.SSLCert.from_der(d) + s = certs.SSLCert.from_der(d) assert s.cn diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index d63ee50e..b0e77ce1 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -1,12 +1,12 @@ import mock from mitmproxy.exceptions import ContentViewException -from netlib.http import Headers -from netlib.http import url -from netlib import multidict +from mitmproxy.net.http import Headers +from mitmproxy.net.http import url +from mitmproxy.types import multidict import mitmproxy.contentviews as cv from . import tutils -import netlib.tutils +import mitmproxy.test.tutils try: import pyamf @@ -232,7 +232,7 @@ def test_get_content_view(): def test_get_message_content_view(): - r = netlib.tutils.treq() + r = mitmproxy.test.tutils.treq() desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) assert desc == "Raw" diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index 071638a9..3bcb7013 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -9,7 +9,7 @@ import queue from mitmproxy.exceptions import Kill, ControlException from mitmproxy import proxy from mitmproxy import master -from netlib.tutils import raises +from mitmproxy.test.tutils import raises class TMsg: diff --git a/test/mitmproxy/test_custom_contentview.py b/test/mitmproxy/test_custom_contentview.py index 889fb8b3..28f7fb33 100644 --- a/test/mitmproxy/test_custom_contentview.py +++ b/test/mitmproxy/test_custom_contentview.py @@ -1,5 +1,5 @@ import mitmproxy.contentviews as cv -from netlib.http import Headers +from mitmproxy.net.http import Headers def test_custom_views(): diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 7551c1c8..6972da0c 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -6,17 +6,16 @@ from mitmproxy import options from mitmproxy import contentviews from mitmproxy import proxy from mitmproxy.addons import script +from mitmproxy.utils import data from mitmproxy import master -import netlib.utils - -from netlib import tutils as netutils -from netlib.http import Headers -from netlib.http import cookies +from mitmproxy.test import tutils as netutils +from mitmproxy.net.http import Headers +from mitmproxy.net.http import cookies from . import tutils, mastertest -example_dir = netlib.utils.Data(__name__).push("../../examples") +example_dir = data.Data(__name__).push("../../examples") class ScriptError(Exception): diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 9ebcca34..5b9f3835 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -1,8 +1,8 @@ import mock import io -import netlib.utils -from netlib.http import Headers +import mitmproxy.test.tutils +from mitmproxy.net.http import Headers import mitmproxy.io from mitmproxy import flowfilter, options from mitmproxy.addons import state @@ -59,7 +59,7 @@ class TestHTTPFlow: def test_backup(self): f = tutils.tflow() - f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) f.request.content = b"foo" assert not f.modified() f.backup() @@ -212,7 +212,7 @@ class TestState: assert c.add_flow(newf) assert c.active_flow_count() == 2 - f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) assert c.update_flow(f) assert c.flow_count() == 2 assert c.active_flow_count() == 1 @@ -220,7 +220,7 @@ class TestState: assert not c.update_flow(None) assert c.active_flow_count() == 1 - newf.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + newf.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) assert c.update_flow(newf) assert c.active_flow_count() == 0 @@ -252,7 +252,7 @@ class TestState: c.set_view_filter("~s") assert c.filter_txt == "~s" assert len(c.view) == 0 - f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) c.update_flow(f) assert len(c.view) == 1 c.set_view_filter(None) @@ -284,7 +284,7 @@ class TestState: def _add_response(self, state): f = tutils.tflow() state.add_flow(f) - f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) state.update_flow(f) def _add_error(self, state): @@ -444,11 +444,11 @@ class TestFlowMaster: fm.addons.add(s) f = tutils.tflow(req=None) fm.clientconnect(f.client_conn) - f.request = http.HTTPRequest.wrap(netlib.tutils.treq()) + f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) fm.request(f) assert s.flow_count() == 1 - f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) fm.response(f) assert s.flow_count() == 1 @@ -473,7 +473,7 @@ class TestRequest: assert r.get_state() == r2.get_state() def test_get_url(self): - r = http.HTTPRequest.wrap(netlib.tutils.treq()) + r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) assert r.url == "http://address:22/path" @@ -494,7 +494,7 @@ class TestRequest: assert r.pretty_url == "https://foo.com:22/path" def test_replace(self): - r = http.HTTPRequest.wrap(netlib.tutils.treq()) + r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) r.path = "path/foo" r.headers["Foo"] = "fOo" r.content = b"afoob" @@ -504,7 +504,7 @@ class TestRequest: assert r.headers["boo"] == "boo" def test_constrain_encoding(self): - r = http.HTTPRequest.wrap(netlib.tutils.treq()) + r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) r.headers["accept-encoding"] = "gzip, oink" r.constrain_encoding() assert "oink" not in r.headers["accept-encoding"] @@ -514,7 +514,7 @@ class TestRequest: assert "oink" not in r.headers["accept-encoding"] def test_get_content_type(self): - resp = http.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) resp.headers = Headers(content_type="text/plain") assert resp.headers["content-type"] == "text/plain" @@ -528,7 +528,7 @@ class TestResponse: assert resp2.get_state() == resp.get_state() def test_replace(self): - r = http.HTTPResponse.wrap(netlib.tutils.tresp()) + r = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) r.headers["Foo"] = "fOo" r.content = b"afoob" assert r.replace("foo(?i)", "boo") == 3 @@ -536,7 +536,7 @@ class TestResponse: assert r.headers["boo"] == "boo" def test_get_content_type(self): - resp = http.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) resp.headers = Headers(content_type="text/plain") assert resp.headers["content-type"] == "text/plain" diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index aafd5a1c..df0ccb77 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -1,7 +1,7 @@ import re -import netlib.tutils -from netlib.http import Headers +import mitmproxy.test.tutils +from mitmproxy.net.http import Headers from mitmproxy import export # heh from . import tutils @@ -20,15 +20,15 @@ def python_equals(testdata, text): def req_get(): - return netlib.tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz") + return mitmproxy.test.tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz") def req_post(): - return netlib.tutils.treq(method=b'POST', headers=()) + return mitmproxy.test.tutils.treq(method=b'POST', headers=()) def req_patch(): - return netlib.tutils.treq(method=b'PATCH', path=b"/path?query=param") + return mitmproxy.test.tutils.treq(method=b'PATCH', path=b"/path?query=param") class TestExportCurlCommand: diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 1d677bd3..3c845707 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -2,7 +2,7 @@ import copy from mitmproxy import optmanager from mitmproxy import exceptions -from netlib import tutils +from mitmproxy.test import tutils class TO(optmanager.OptManager): diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 7d401184..a9d9bb87 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -8,9 +8,9 @@ from mitmproxy.proxy import ProxyConfig from mitmproxy import connections from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler from mitmproxy.proxy import config -from netlib.exceptions import TcpDisconnect +from mitmproxy import exceptions from pathod import test -from netlib.http import http1 +from mitmproxy.net.http import http1 from . import tutils @@ -40,7 +40,7 @@ class TestServerConnection: sc.connect() sc.connection = mock.Mock() sc.connection.recv = mock.Mock(return_value=False) - sc.connection.flush = mock.Mock(side_effect=TcpDisconnect) + sc.connection.flush = mock.Mock(side_effect=exceptions.TcpDisconnect) sc.finish() self.d.shutdown() diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 93a82954..7dd738f4 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -2,20 +2,23 @@ import os import socket import time -import netlib.tutils +import mitmproxy.test.tutils from mitmproxy import controller from mitmproxy import options from mitmproxy.addons import script from mitmproxy import http from mitmproxy.proxy.config import HostMatcher, parse_server_spec -import netlib.http -from netlib import tcp, socks -from netlib.certutils import SSLCert -from netlib.exceptions import HttpReadDisconnect, HttpException -from netlib.http import authentication, http1 -from netlib.tcp import Address -from netlib.tutils import raises -from pathod import pathoc, pathod +import mitmproxy.net.http +from mitmproxy.net import tcp +from mitmproxy.net import socks +from mitmproxy import certs +from mitmproxy import exceptions +from mitmproxy.net.http import authentication +from mitmproxy.net.http import http1 +from mitmproxy.net.tcp import Address +from mitmproxy.test.tutils import raises +from pathod import pathoc +from pathod import pathod from . import tutils, tservers @@ -144,9 +147,9 @@ class TcpMixin: # Test that we get the original SSL cert if self.ssl: - i_cert = SSLCert(i.sslinfo.certchain[0]) - i2_cert = SSLCert(i2.sslinfo.certchain[0]) - n_cert = SSLCert(n.sslinfo.certchain[0]) + i_cert = certs.SSLCert(i.sslinfo.certchain[0]) + i2_cert = certs.SSLCert(i2.sslinfo.certchain[0]) + n_cert = certs.SSLCert(n.sslinfo.certchain[0]) assert i_cert == i2_cert assert i_cert != n_cert @@ -156,7 +159,7 @@ class TcpMixin: # mitmproxy responds with bad gateway assert self.pathod(spec).status_code == 502 self._ignore_on() - with raises(HttpException): + with raises(exceptions.HttpException): self.pathod(spec) # pathoc tries to parse answer as HTTP self._ignore_off() @@ -190,9 +193,9 @@ class TcpMixin: # Test that we get the original SSL cert if self.ssl: - i_cert = SSLCert(i.sslinfo.certchain[0]) - i2_cert = SSLCert(i2.sslinfo.certchain[0]) - n_cert = SSLCert(n.sslinfo.certchain[0]) + i_cert = certs.SSLCert(i.sslinfo.certchain[0]) + i2_cert = certs.SSLCert(i2.sslinfo.certchain[0]) + n_cert = certs.SSLCert(n.sslinfo.certchain[0]) assert i_cert == i2_cert == n_cert @@ -294,7 +297,7 @@ class TestHTTPAuth(tservers.HTTPProxyTest): h'%s'='%s' """ % ( self.server.port, - netlib.http.authentication.BasicProxyAuth.AUTH_HEADER, + mitmproxy.net.http.authentication.BasicProxyAuth.AUTH_HEADER, authentication.assemble_http_basic_auth("basic", "test", "test") )) assert ret.status_code == 202 @@ -311,7 +314,7 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest): '/p/202' h'%s'='%s' """ % ( - netlib.http.authentication.BasicWebsiteAuth.AUTH_HEADER, + mitmproxy.net.http.authentication.BasicWebsiteAuth.AUTH_HEADER, authentication.assemble_http_basic_auth("basic", "test", "test") )) assert ret.status_code == 202 @@ -435,7 +438,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): def test_verification_w_bad_cert(self): # We only test for a single invalid cert here. # Actual testing of different root-causes (invalid hostname, expired, ...) - # is done in netlib. + # is done in mitmproxy.net. self.config.options.ssl_insecure = False r = self._request() assert r.status_code == 502 @@ -791,7 +794,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): class MasterFakeResponse(tservers.TestMaster): @controller.handler def request(self, f): - f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) class TestFakeResponse(tservers.HTTPProxyTest): @@ -830,7 +833,7 @@ class TestKillRequest(tservers.HTTPProxyTest): masterclass = MasterKillRequest def test_kill(self): - with raises(HttpReadDisconnect): + with raises(exceptions.HttpReadDisconnect): self.pathod("200") # Nothing should have hit the server assert not self.server.last_log() @@ -847,7 +850,7 @@ class TestKillResponse(tservers.HTTPProxyTest): masterclass = MasterKillResponse def test_kill(self): - with raises(HttpReadDisconnect): + with raises(exceptions.HttpReadDisconnect): self.pathod("200") # The server should have seen a request assert self.server.last_log() @@ -870,7 +873,7 @@ class MasterIncomplete(tservers.TestMaster): @controller.handler def request(self, f): - resp = http.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) resp.content = None f.response = resp @@ -1050,7 +1053,7 @@ class AddUpstreamCertsToClientChainMixin: def test_add_upstream_certs_to_client_chain(self): with open(self.servercert, "rb") as f: d = f.read() - upstreamCert = SSLCert.from_pem(d) + upstreamCert = certs.SSLCert.from_pem(d) p = self.pathoc() with p.connect(): upstream_cert_found_in_client_chain = False diff --git a/test/mitmproxy/test_types_bidi.py b/test/mitmproxy/test_types_bidi.py new file mode 100644 index 00000000..0494ac9d --- /dev/null +++ b/test/mitmproxy/test_types_bidi.py @@ -0,0 +1,11 @@ +from mitmproxy.types import bidi +from mitmproxy.test import tutils + + +def test_bidi(): + b = bidi.BiDi(a=1, b=2) + assert b.a == 1 + assert b.get_name(1) == "a" + assert b.get_name(5) is None + tutils.raises(AttributeError, getattr, b, "c") + tutils.raises(ValueError, bidi.BiDi, one=1, two=1) diff --git a/test/netlib/test_multidict.py b/test/mitmproxy/test_types_multidict.py index a9523fd9..d566905c 100644 --- a/test/netlib/test_multidict.py +++ b/test/mitmproxy/test_types_multidict.py @@ -1,5 +1,5 @@ -from netlib import tutils -from netlib.multidict import MultiDict, ImmutableMultiDict, MultiDictView +from mitmproxy.test import tutils +from mitmproxy.types import multidict class _TMulti: @@ -8,11 +8,11 @@ class _TMulti: return key.lower() -class TMultiDict(_TMulti, MultiDict): +class TMultiDict(_TMulti, multidict.MultiDict): pass -class TImmutableMultiDict(_TMulti, ImmutableMultiDict): +class TImmutableMultiDict(_TMulti, multidict.ImmutableMultiDict): pass @@ -238,7 +238,7 @@ class TParent: class TestMultiDictView: def test_modify(self): p = TParent() - tv = MultiDictView(p.getter, p.setter) + tv = multidict.MultiDictView(p.getter, p.setter) assert len(tv) == 0 tv["a"] = "b" assert p.vals == (("a", "b"),) diff --git a/test/netlib/test_basetypes.py b/test/mitmproxy/test_types_serializable.py index aa415784..dd4a3778 100644 --- a/test/netlib/test_basetypes.py +++ b/test/mitmproxy/test_types_serializable.py @@ -1,7 +1,7 @@ -from netlib import basetypes +from mitmproxy.types import serializable -class SerializableDummy(basetypes.Serializable): +class SerializableDummy(serializable.Serializable): def __init__(self, i): self.i = i diff --git a/test/mitmproxy/test_utils_data.py b/test/mitmproxy/test_utils_data.py new file mode 100644 index 00000000..c6e4420e --- /dev/null +++ b/test/mitmproxy/test_utils_data.py @@ -0,0 +1,7 @@ +from mitmproxy.utils import data +from . import tutils + + +def test_pkg_data(): + assert data.pkg_data.path("tools/console") + tutils.raises("does not exist", data.pkg_data.path, "nonexistent") diff --git a/test/netlib/test_debug.py b/test/mitmproxy/test_utils_debug.py index bdb85c9e..9acf8192 100644 --- a/test/netlib/test_debug.py +++ b/test/mitmproxy/test_utils_debug.py @@ -1,6 +1,6 @@ import io -from netlib import debug +from mitmproxy.utils import debug def test_dump_info(): diff --git a/test/netlib/test_human.py b/test/mitmproxy/test_utils_human.py index bb97dc54..443c8f66 100644 --- a/test/netlib/test_human.py +++ b/test/mitmproxy/test_utils_human.py @@ -1,5 +1,6 @@ import time -from netlib import human, tutils +from mitmproxy.utils import human +from mitmproxy.test import tutils def test_format_timestamp(): diff --git a/test/mitmproxy/test_utils.py b/test/mitmproxy/test_utils_lrucache.py index ed59f484..07b96b4d 100644 --- a/test/mitmproxy/test_utils.py +++ b/test/mitmproxy/test_utils_lrucache.py @@ -1,16 +1,8 @@ -from mitmproxy import utils -from . import tutils - -utils.CERT_SLEEP_TIME = 0 - - -def test_pkg_data(): - assert utils.pkg_data.path("tools/console") - tutils.raises("does not exist", utils.pkg_data.path, "nonexistent") +from mitmproxy.utils import lrucache def test_LRUCache(): - cache = utils.LRUCache(2) + cache = lrucache.LRUCache(2) class Foo: ran = False diff --git a/test/netlib/test_strutils.py b/test/mitmproxy/test_utils_strutils.py index 36f709da..d4c2883c 100644 --- a/test/netlib/test_strutils.py +++ b/test/mitmproxy/test_utils_strutils.py @@ -1,4 +1,5 @@ -from netlib import strutils, tutils +from mitmproxy.utils import strutils +from mitmproxy.test import tutils def test_always_bytes(): diff --git a/test/netlib/test_version_check.py b/test/mitmproxy/test_utils_version_check.py index 01820dad..5c8d8c8c 100644 --- a/test/netlib/test_version_check.py +++ b/test/mitmproxy/test_utils_version_check.py @@ -1,6 +1,6 @@ import io import mock -from netlib import version_check +from mitmproxy.utils import version_check @mock.patch("sys.exit") diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 1243bca0..e07102e1 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -10,8 +10,9 @@ from mitmproxy import master from mitmproxy.addons import state import pathod.test import pathod.pathoc -from mitmproxy import controller, options -import netlib.exceptions +from mitmproxy import controller +from mitmproxy import options +from mitmproxy import exceptions class TestMaster(master.Master): @@ -98,7 +99,7 @@ class ProxyTestBase: def teardown(self): try: self.server.wait_for_silence() - except netlib.exceptions.Timeout: + except exceptions.Timeout: # FIXME: Track down the Windows sync issues if sys.platform != "win32": raise diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index eb0c90e9..c83223f6 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -9,13 +9,13 @@ from unittest.case import SkipTest import io -import netlib.utils -import netlib.tutils +import mitmproxy.test.tutils from mitmproxy import controller from mitmproxy import connections from mitmproxy import flow from mitmproxy import http from mitmproxy import tcp +from mitmproxy.utils import data def _skip_windows(*args): @@ -102,9 +102,9 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): if server_conn is True: server_conn = tserver_conn() if req is True: - req = netlib.tutils.treq() + req = mitmproxy.test.tutils.treq() if resp is True: - resp = netlib.tutils.tresp() + resp = mitmproxy.test.tutils.tresp() if err is True: err = terr() @@ -197,7 +197,7 @@ class MockParser(argparse.ArgumentParser): raise Exception(message) -raises = netlib.tutils.raises +raises = mitmproxy.test.tutils.raises @contextmanager @@ -208,4 +208,4 @@ def capture_stderr(command, *args, **kwargs): sys.stderr = out -test_data = netlib.utils.Data(__name__) +test_data = data.Data(__name__) diff --git a/test/netlib/test_utils.py b/test/netlib/test_utils.py deleted file mode 100644 index 9dcbffd8..00000000 --- a/test/netlib/test_utils.py +++ /dev/null @@ -1,19 +0,0 @@ -# coding=utf-8 - -from netlib import utils, tutils - - -def test_is_valid_host(): - assert not utils.is_valid_host(b"") - assert utils.is_valid_host(b"one.two") - assert not utils.is_valid_host(b"one" * 255) - assert utils.is_valid_host(b"one.two.") - - -def test_bidi(): - b = utils.BiDi(a=1, b=2) - assert b.a == 1 - assert b.get_name(1) == "a" - assert b.get_name(5) is None - tutils.raises(AttributeError, getattr, b, "c") - tutils.raises(ValueError, utils.BiDi, one=1, two=1) diff --git a/test/pathod/test_language_http2.py b/test/pathod/test_language_http2.py index 11e100ac..7d3a8e60 100644 --- a/test/pathod/test_language_http2.py +++ b/test/pathod/test_language_http2.py @@ -1,7 +1,7 @@ import io -from netlib import tcp -from netlib.http import user_agents +from mitmproxy.net import tcp +from mitmproxy.net.http import user_agents from pathod import language from pathod.language import http2 diff --git a/test/pathod/test_language_websocket.py b/test/pathod/test_language_websocket.py index 9d533d98..49fbd4e6 100644 --- a/test/pathod/test_language_websocket.py +++ b/test/pathod/test_language_websocket.py @@ -1,6 +1,6 @@ from pathod import language from pathod.language import websockets -import netlib.websockets +import mitmproxy.net.websockets from . import tutils @@ -62,7 +62,7 @@ class TestWebsocketFrame: def test_flags(self): wf = parse_request("wf:fin:mask:rsv1:rsv2:rsv3") - frm = netlib.websockets.Frame.from_bytes(tutils.render(wf)) + frm = mitmproxy.net.websockets.Frame.from_bytes(tutils.render(wf)) assert frm.header.fin assert frm.header.mask assert frm.header.rsv1 @@ -70,7 +70,7 @@ class TestWebsocketFrame: assert frm.header.rsv3 wf = parse_request("wf:-fin:-mask:-rsv1:-rsv2:-rsv3") - frm = netlib.websockets.Frame.from_bytes(tutils.render(wf)) + frm = mitmproxy.net.websockets.Frame.from_bytes(tutils.render(wf)) assert not frm.header.fin assert not frm.header.mask assert not frm.header.rsv1 @@ -80,15 +80,15 @@ class TestWebsocketFrame: def fr(self, spec, **kwargs): settings = language.base.Settings(**kwargs) wf = parse_request(spec) - return netlib.websockets.Frame.from_bytes(tutils.render(wf, settings)) + return mitmproxy.net.websockets.Frame.from_bytes(tutils.render(wf, settings)) def test_construction(self): assert self.fr("wf:c1").header.opcode == 1 assert self.fr("wf:c0").header.opcode == 0 assert self.fr("wf:cbinary").header.opcode ==\ - netlib.websockets.OPCODE.BINARY + mitmproxy.net.websockets.OPCODE.BINARY assert self.fr("wf:ctext").header.opcode ==\ - netlib.websockets.OPCODE.TEXT + mitmproxy.net.websockets.OPCODE.TEXT def test_rawbody(self): frm = self.fr("wf:mask:r'foo'") diff --git a/test/pathod/test_log.py b/test/pathod/test_log.py index deb0f613..8890e7d9 100644 --- a/test/pathod/test_log.py +++ b/test/pathod/test_log.py @@ -1,7 +1,7 @@ import io from pathod import log -from netlib.exceptions import TcpDisconnect +from mitmproxy import exceptions class DummyIO(io.StringIO): @@ -20,6 +20,6 @@ def test_disconnect(): try: with l.ctx() as lg: lg("Test") - except TcpDisconnect: + except exceptions.TcpDisconnect: pass assert "Test" in outf.getvalue() diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index f9670d73..90177ff6 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -1,11 +1,11 @@ import io from mock import Mock -from netlib import http -from netlib import tcp -from netlib.exceptions import NetlibException -from netlib.http import http1 -from netlib.tutils import raises +from mitmproxy.net import http +from mitmproxy.net import tcp +from mitmproxy.net.http import http1 +from mitmproxy.test.tutils import raises +from mitmproxy import exceptions from pathod import pathoc, language from pathod.protocols.http2 import HTTP2StateProtocol @@ -36,7 +36,7 @@ class PathocTestDaemon(tutils.DaemonTests): r = r.freeze(language.Settings()) try: c.request(r) - except NetlibException: + except exceptions.NetlibException: pass self.d.wait_for_silence() return s.getvalue() diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index 89d7c562..8ee7a2fd 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -1,8 +1,8 @@ import io from pathod import pathod -from netlib import tcp -from netlib.exceptions import HttpException, TlsException +from mitmproxy.net import tcp +from mitmproxy import exceptions from . import tutils @@ -157,7 +157,7 @@ class CommonTests(tutils.DaemonTests): def test_invalid_content_length(self): tutils.raises( - HttpException, + exceptions.HttpException, self.pathoc, ["get:/:h'content-length'='foo'"] ) @@ -166,7 +166,7 @@ class CommonTests(tutils.DaemonTests): assert "Unparseable Content Length" in l["msg"] def test_invalid_headers(self): - tutils.raises(HttpException, self.pathoc, ["get:/:h'\t'='foo'"]) + tutils.raises(exceptions.HttpException, self.pathoc, ["get:/:h'\t'='foo'"]) l = self.d.last_log() assert l["type"] == "error" assert "Invalid headers" in l["msg"] @@ -225,7 +225,7 @@ class TestDaemon(CommonTests): def test_connect_err(self): tutils.raises( - HttpException, + exceptions.HttpException, self.pathoc, [r"get:'http://foo.com/p/202':da"], connect_to=("localhost", self.d.port) @@ -241,7 +241,7 @@ class TestDaemonSSL(CommonTests): c.wbufsize = 0 with c.connect(): c.wfile.write(b"\0\0\0\0") - tutils.raises(TlsException, c.convert_to_ssl) + tutils.raises(exceptions.TlsException, c.convert_to_ssl) l = self.d.last_log() assert l["type"] == "error" assert "SSL" in l["msg"] diff --git a/test/pathod/test_protocols_http2.py b/test/pathod/test_protocols_http2.py index 7300cc1d..d77702a3 100644 --- a/test/pathod/test_protocols_http2.py +++ b/test/pathod/test_protocols_http2.py @@ -2,12 +2,12 @@ import mock import codecs import hyperframe -from netlib import tcp, http -from netlib.tutils import raises -from netlib.exceptions import TcpDisconnect -from netlib.http import http2 +from mitmproxy.net import tcp, http +from mitmproxy.test.tutils import raises +from mitmproxy.net.http import http2 +from mitmproxy import exceptions -from ..netlib import tservers as netlib_tservers +from ..mitmproxy.net import tservers as net_tservers from pathod.protocols.http2 import HTTP2StateProtocol, TCPHandler @@ -66,7 +66,7 @@ class TestProtocol: assert mock_server_method.called -class TestCheckALPNMatch(netlib_tservers.ServerTestBase): +class TestCheckALPNMatch(net_tservers.ServerTestBase): handler = EchoHandler ssl = dict( alpn_select=b'h2', @@ -82,7 +82,7 @@ class TestCheckALPNMatch(netlib_tservers.ServerTestBase): assert protocol.check_alpn() -class TestCheckALPNMismatch(netlib_tservers.ServerTestBase): +class TestCheckALPNMismatch(net_tservers.ServerTestBase): handler = EchoHandler ssl = dict( alpn_select=None, @@ -99,7 +99,7 @@ class TestCheckALPNMismatch(netlib_tservers.ServerTestBase): protocol.check_alpn() -class TestPerformServerConnectionPreface(netlib_tservers.ServerTestBase): +class TestPerformServerConnectionPreface(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): @@ -132,11 +132,11 @@ class TestPerformServerConnectionPreface(netlib_tservers.ServerTestBase): protocol.perform_server_connection_preface() assert protocol.connection_preface_performed - with raises(TcpDisconnect): + with raises(exceptions.TcpDisconnect): protocol.perform_server_connection_preface(force=True) -class TestPerformClientConnectionPreface(netlib_tservers.ServerTestBase): +class TestPerformClientConnectionPreface(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): @@ -197,7 +197,7 @@ class TestserverstreamIds: assert self.protocol.current_stream_id == 6 -class TestApplySettings(netlib_tservers.ServerTestBase): +class TestApplySettings(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): # check settings acknowledgement @@ -290,7 +290,7 @@ class TestCreateBody: assert bytes[2] == codecs.decode('0000020001000000013432', 'hex_codec') -class TestReadRequest(netlib_tservers.ServerTestBase): +class TestReadRequest(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): @@ -320,7 +320,7 @@ class TestReadRequest(netlib_tservers.ServerTestBase): assert req.content == b'foobar' -class TestReadRequestRelative(netlib_tservers.ServerTestBase): +class TestReadRequestRelative(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( @@ -343,7 +343,7 @@ class TestReadRequestRelative(netlib_tservers.ServerTestBase): assert req.path == "*" -class TestReadRequestAbsolute(netlib_tservers.ServerTestBase): +class TestReadRequestAbsolute(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( @@ -367,7 +367,7 @@ class TestReadRequestAbsolute(netlib_tservers.ServerTestBase): assert req.port == 22 -class TestReadResponse(netlib_tservers.ServerTestBase): +class TestReadResponse(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( @@ -396,7 +396,7 @@ class TestReadResponse(netlib_tservers.ServerTestBase): assert resp.timestamp_end -class TestReadEmptyResponse(netlib_tservers.ServerTestBase): +class TestReadEmptyResponse(net_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index c2243578..ccc3df43 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -5,9 +5,10 @@ import requests import io import urllib -from netlib import tcp -from netlib import utils -from netlib import tutils +from mitmproxy.utils import data + +from mitmproxy.net import tcp +from mitmproxy.test import tutils from pathod import language from pathod import pathoc @@ -142,7 +143,7 @@ tmpdir = tutils.tmpdir raises = tutils.raises -test_data = utils.Data(__name__) +test_data = data.Data(__name__) def render(r, settings=language.Settings()): @@ -19,4 +19,4 @@ commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html [testenv:lint] deps = flake8>=2.6.2, <3.1 -commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test +commands = flake8 --jobs 8 --count mitmproxy pathod examples test |