aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNikhil Soni <krsoninikhil@gmail.com>2017-03-03 12:58:44 +0530
committerGitHub <noreply@github.com>2017-03-03 12:58:44 +0530
commit0081d9b82807b178bd6d00ca250d38aeeeed2d33 (patch)
tree8d244a7b9ade13f0e1836f8eb9fbc534a1cc662d
parent3da8532bed3305b01e3f3ab556f9dbc652177c6b (diff)
parentbae4cdf8d5cc434938c74a041f762075513dd8e4 (diff)
downloadmitmproxy-0081d9b82807b178bd6d00ca250d38aeeeed2d33.tar.gz
mitmproxy-0081d9b82807b178bd6d00ca250d38aeeeed2d33.tar.bz2
mitmproxy-0081d9b82807b178bd6d00ca250d38aeeeed2d33.zip
Merge branch 'master' into on-issues
-rw-r--r--docs/certinstall.rst12
-rw-r--r--examples/complex/har_dump.py10
-rwxr-xr-xexamples/complex/xss_scanner.py407
-rw-r--r--mitmproxy/certs.py21
-rw-r--r--mitmproxy/connections.py57
-rw-r--r--mitmproxy/flowfilter.py11
-rw-r--r--mitmproxy/http.py4
-rw-r--r--mitmproxy/io_compat.py8
-rw-r--r--mitmproxy/master.py4
-rw-r--r--mitmproxy/net/check.py14
-rw-r--r--mitmproxy/net/socks.py18
-rw-r--r--mitmproxy/net/tcp.py194
-rw-r--r--mitmproxy/net/wsgi.py7
-rw-r--r--mitmproxy/proxy/config.py5
-rw-r--r--mitmproxy/proxy/protocol/base.py10
-rw-r--r--mitmproxy/proxy/protocol/http.py18
-rw-r--r--mitmproxy/proxy/protocol/http2.py5
-rw-r--r--mitmproxy/proxy/protocol/tls.py7
-rw-r--r--mitmproxy/proxy/root_context.py4
-rw-r--r--mitmproxy/proxy/server.py5
-rw-r--r--mitmproxy/test/tflow.py16
-rw-r--r--mitmproxy/tools/console/flowdetailview.py6
-rw-r--r--mitmproxy/tools/console/flowview.py2
-rw-r--r--mitmproxy/tools/console/master.py8
-rw-r--r--mitmproxy/tools/console/palettepicker.py4
-rw-r--r--mitmproxy/tools/console/statusbar.py8
-rw-r--r--mitmproxy/tools/dump.py2
-rw-r--r--mitmproxy/tools/web/app.py1
-rw-r--r--mitmproxy/tools/web/master.py2
-rw-r--r--mitmproxy/tools/web/static/app.js6
-rw-r--r--pathod/pathoc.py4
-rw-r--r--pathod/pathod.py2
-rw-r--r--pathod/protocols/http2.py6
-rw-r--r--pathod/test.py6
-rw-r--r--setup.cfg10
-rw-r--r--setup.py1
-rw-r--r--test/mitmproxy/addons/test_dumper.py4
-rwxr-xr-xtest/mitmproxy/examples/test_xss_scanner.py368
-rw-r--r--test/mitmproxy/net/test_check.py1
-rw-r--r--test/mitmproxy/net/test_socks.py5
-rw-r--r--test/mitmproxy/net/test_tcp.py24
-rw-r--r--test/mitmproxy/net/tservers.py4
-rw-r--r--test/mitmproxy/proxy/protocol/test_http2.py93
-rw-r--r--test/mitmproxy/proxy/protocol/test_websocket.py8
-rw-r--r--test/mitmproxy/proxy/test_server.py9
-rw-r--r--test/mitmproxy/test_certs.py27
-rw-r--r--test/mitmproxy/test_connections.py211
-rw-r--r--test/mitmproxy/test_eventsequence.py2
-rw-r--r--test/mitmproxy/test_flow.py247
-rw-r--r--test/mitmproxy/test_flowfilter.py109
-rw-r--r--test/mitmproxy/test_http.py257
-rw-r--r--test/mitmproxy/test_optmanager.py22
-rw-r--r--test/mitmproxy/test_proxy.py47
-rw-r--r--test/mitmproxy/tservers.py5
-rw-r--r--web/src/js/components/FlowView/Details.jsx2
-rw-r--r--web/src/js/filt/filt.peg4
56 files changed, 1674 insertions, 680 deletions
diff --git a/docs/certinstall.rst b/docs/certinstall.rst
index 5d97e92c..1bd6df99 100644
--- a/docs/certinstall.rst
+++ b/docs/certinstall.rst
@@ -42,7 +42,7 @@ iOS
See http://jasdev.me/intercepting-ios-traffic
-and http://web.archive.org/web/20150920082614/http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
+and https://web.archive.org/web/20150920082614/http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377
iOS Simulator
^^^^^^^^^^^^^
@@ -52,7 +52,7 @@ See https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore
Java
^^^^
-See http://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html
+See https://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html
Android/Android Simulator
^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -62,7 +62,7 @@ See http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets
Windows
^^^^^^^
-See http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7
+See https://web.archive.org/web/20160612045445/http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7
Windows (automated)
^^^^^^^^^^^^^^^^^^^
@@ -79,7 +79,7 @@ See https://support.apple.com/kb/PH7297?locale=en_US
Ubuntu/Debian
^^^^^^^^^^^^^
-See http://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861
+See https://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861
Mozilla Firefox
^^^^^^^^^^^^^^^
@@ -89,7 +89,7 @@ See https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox
Chrome on Linux
^^^^^^^^^^^^^^^
-See https://code.google.com/p/chromium/wiki/LinuxCertManagement
+See https://stackoverflow.com/a/15076602/198996
The mitmproxy certificate authority
@@ -205,4 +205,4 @@ directory and uses this as the client cert.
-.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/
+.. _Certificate Pinning: https://security.stackexchange.com/questions/29988/what-is-certificate-pinning/
diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py
index f7c1e658..51983b54 100644
--- a/examples/complex/har_dump.py
+++ b/examples/complex/har_dump.py
@@ -10,7 +10,7 @@ import zlib
import os
from datetime import datetime
-import pytz
+from datetime import timezone
import mitmproxy
@@ -89,7 +89,7 @@ def response(flow):
# Timings set to -1 will be ignored as per spec.
full_time = sum(v for v in timings.values() if v > -1)
- started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
+ started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, timezone.utc).isoformat()
# Response body size and encoding
response_body_size = len(flow.response.raw_content)
@@ -173,10 +173,6 @@ def done():
mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
-def format_datetime(dt):
- return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
-
-
def format_cookies(cookie_list):
rv = []
@@ -198,7 +194,7 @@ def format_cookies(cookie_list):
# Expiration time needs to be formatted
expire_ts = cookies.get_expiration_ts(attrs)
if expire_ts is not None:
- cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
+ cookie_har["expires"] = datetime.fromtimestamp(expire_ts, timezone.utc).isoformat()
rv.append(cookie_har)
diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py
new file mode 100755
index 00000000..a0572d5d
--- /dev/null
+++ b/examples/complex/xss_scanner.py
@@ -0,0 +1,407 @@
+"""
+
+ __ __ _____ _____ _____
+ \ \ / // ____/ ____| / ____|
+ \ V /| (___| (___ | (___ ___ __ _ _ __ _ __ ___ _ __
+ > < \___ \\___ \ \___ \ / __/ _` | '_ \| '_ \ / _ \ '__|
+ / . \ ____) |___) | ____) | (_| (_| | | | | | | | __/ |
+ /_/ \_\_____/_____/ |_____/ \___\__,_|_| |_|_| |_|\___|_|
+
+
+This script automatically scans all visited webpages for XSS and SQLi vulnerabilities.
+
+Usage: mitmproxy -s xss_scanner.py
+
+This script scans for vulnerabilities by injecting a fuzzing payload (see PAYLOAD below) into 4 different places
+and examining the HTML to look for XSS and SQLi injection vulnerabilities. The XSS scanning functionality works by
+looking to see whether it is possible to inject HTML based off of of where the payload appears in the page and what
+characters are escaped. In addition, it also looks for any script tags that load javascript from unclaimed domains.
+The SQLi scanning functionality works by using regular expressions to look for errors from a number of different
+common databases. Since it is only looking for errors, it will not find blind SQLi vulnerabilities.
+
+The 4 places it injects the payload into are:
+1. URLs (e.g. https://example.com/ -> https://example.com/PAYLOAD/)
+2. Queries (e.g. https://example.com/index.html?a=b -> https://example.com/index.html?a=PAYLOAD)
+3. Referers (e.g. The referer changes from https://example.com to PAYLOAD)
+4. User Agents (e.g. The UA changes from Chrome to PAYLOAD)
+
+Reports from this script show up in the event log (viewable by pressing e) and formatted like:
+
+===== XSS Found ====
+XSS URL: http://daviddworken.com/vulnerableUA.php
+Injection Point: User Agent
+Suggested Exploit: <script>alert(0)</script>
+Line: 1029zxcs'd"ao<ac>so[sb]po(pc)se;sl/bsl\eq=3847asd
+
+"""
+
+from mitmproxy import ctx
+from socket import gaierror, gethostbyname
+from urllib.parse import urlparse
+import requests
+import re
+from html.parser import HTMLParser
+from mitmproxy import http
+from typing import Dict, Union, Tuple, Optional, List, NamedTuple
+
+# The actual payload is put between a frontWall and a backWall to make it easy
+# to locate the payload with regular expressions
+FRONT_WALL = b"1029zxc"
+BACK_WALL = b"3847asd"
+PAYLOAD = b"""s'd"ao<ac>so[sb]po(pc)se;sl/bsl\\eq="""
+FULL_PAYLOAD = FRONT_WALL + PAYLOAD + BACK_WALL
+
+# A XSSData is a named tuple with the following fields:
+# - url -> str
+# - injection_point -> str
+# - exploit -> str
+# - line -> str
+XSSData = NamedTuple('XSSData', [('url', str),
+ ('injection_point', str),
+ ('exploit', str),
+ ('line', str)])
+
+# A SQLiData is named tuple with the following fields:
+# - url -> str
+# - injection_point -> str
+# - regex -> str
+# - dbms -> str
+SQLiData = NamedTuple('SQLiData', [('url', str),
+ ('injection_point', str),
+ ('regex', str),
+ ('dbms', str)])
+
+
+VulnData = Tuple[Optional[XSSData], Optional[SQLiData]]
+Cookies = Dict[str, str]
+
+
+def get_cookies(flow: http.HTTPFlow) -> Cookies:
+ """ Return a dict going from cookie names to cookie values
+ - Note that it includes both the cookies sent in the original request and
+ the cookies sent by the server """
+ return {name: value for name, value in flow.request.cookies.fields}
+
+
+def find_unclaimed_URLs(body: Union[str, bytes], requestUrl: bytes) -> None:
+ """ Look for unclaimed URLs in script tags and log them if found"""
+ class ScriptURLExtractor(HTMLParser):
+ script_URLs = []
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "script" and "src" in [name for name, value in attrs]:
+ for name, value in attrs:
+ if name == "src":
+ self.script_URLs.append(value)
+
+ parser = ScriptURLExtractor()
+ try:
+ parser.feed(body)
+ except TypeError:
+ parser.feed(body.decode('utf-8'))
+ for url in parser.script_URLs:
+ parser = urlparse(url)
+ domain = parser.netloc
+ try:
+ gethostbyname(domain)
+ except gaierror:
+ ctx.log.error("XSS found in %s due to unclaimed URL \"%s\" in script tag." % (requestUrl, url))
+
+
+def test_end_of_URL_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData:
+ """ Test the given URL for XSS via injection onto the end of the URL and
+ log the XSS if found """
+ parsed_URL = urlparse(request_URL)
+ path = parsed_URL.path
+ if path != "" and path[-1] != "/": # ensure the path ends in a /
+ path += "/"
+ path += FULL_PAYLOAD.decode('utf-8') # the path must be a string while the payload is bytes
+ url = parsed_URL._replace(path=path).geturl()
+ body = requests.get(url, cookies=cookies).text.lower()
+ xss_info = get_XSS_data(body, url, "End of URL")
+ sqli_info = get_SQLi_data(body, original_body, url, "End of URL")
+ return xss_info, sqli_info
+
+
+def test_referer_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData:
+ """ Test the given URL for XSS via injection into the referer and
+ log the XSS if found """
+ body = requests.get(request_URL, headers={'referer': FULL_PAYLOAD}, cookies=cookies).text.lower()
+ xss_info = get_XSS_data(body, request_URL, "Referer")
+ sqli_info = get_SQLi_data(body, original_body, request_URL, "Referer")
+ return xss_info, sqli_info
+
+
+def test_user_agent_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData:
+ """ Test the given URL for XSS via injection into the user agent and
+ log the XSS if found """
+ body = requests.get(request_URL, headers={'User-Agent': FULL_PAYLOAD}, cookies=cookies).text.lower()
+ xss_info = get_XSS_data(body, request_URL, "User Agent")
+ sqli_info = get_SQLi_data(body, original_body, request_URL, "User Agent")
+ return xss_info, sqli_info
+
+
+def test_query_injection(original_body: str, request_URL: str, cookies: Cookies):
+ """ Test the given URL for XSS via injection into URL queries and
+ log the XSS if found """
+ parsed_URL = urlparse(request_URL)
+ query_string = parsed_URL.query
+ # queries is a list of parameters where each parameter is set to the payload
+ queries = [query.split("=")[0] + "=" + FULL_PAYLOAD.decode('utf-8') for query in query_string.split("&")]
+ new_query_string = "&".join(queries)
+ new_URL = parsed_URL._replace(query=new_query_string).geturl()
+ body = requests.get(new_URL, cookies=cookies).text.lower()
+ xss_info = get_XSS_data(body, new_URL, "Query")
+ sqli_info = get_SQLi_data(body, original_body, new_URL, "Query")
+ return xss_info, sqli_info
+
+
+def log_XSS_data(xss_info: Optional[XSSData]) -> None:
+ """ Log information about the given XSS to mitmproxy """
+ # If it is None, then there is no info to log
+ if not xss_info:
+ return
+ ctx.log.error("===== XSS Found ====")
+ ctx.log.error("XSS URL: %s" % xss_info.url)
+ ctx.log.error("Injection Point: %s" % xss_info.injection_point)
+ ctx.log.error("Suggested Exploit: %s" % xss_info.exploit)
+ ctx.log.error("Line: %s" % xss_info.line)
+
+
+def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None:
+ """ Log information about the given SQLi to mitmproxy """
+ if not sqli_info:
+ return
+ ctx.log.error("===== SQLi Found =====")
+ ctx.log.error("SQLi URL: %s" % sqli_info.url.decode('utf-8'))
+ ctx.log.error("Injection Point: %s" % sqli_info.injection_point.decode('utf-8'))
+ ctx.log.error("Regex used: %s" % sqli_info.regex.decode('utf-8'))
+ ctx.log.error("Suspected DBMS: %s" % sqli_info.dbms.decode('utf-8'))
+
+
+def get_SQLi_data(new_body: str, original_body: str, request_URL: str, injection_point: str) -> Optional[SQLiData]:
+ """ Return a SQLiDict if there is a SQLi otherwise return None
+ String String URL String -> (SQLiDict or None) """
+ # Regexes taken from Damn Small SQLi Scanner: https://github.com/stamparm/DSSS/blob/master/dsss.py#L17
+ DBMS_ERRORS = {
+ "MySQL": (r"SQL syntax.*MySQL", r"Warning.*mysql_.*", r"valid MySQL result", r"MySqlClient\."),
+ "PostgreSQL": (r"PostgreSQL.*ERROR", r"Warning.*\Wpg_.*", r"valid PostgreSQL result", r"Npgsql\."),
+ "Microsoft SQL Server": (r"Driver.* SQL[\-\_\ ]*Server", r"OLE DB.* SQL Server", r"(\W|\A)SQL Server.*Driver",
+ r"Warning.*mssql_.*", r"(\W|\A)SQL Server.*[0-9a-fA-F]{8}",
+ r"(?s)Exception.*\WSystem\.Data\.SqlClient\.", r"(?s)Exception.*\WRoadhouse\.Cms\."),
+ "Microsoft Access": (r"Microsoft Access Driver", r"JET Database Engine", r"Access Database Engine"),
+ "Oracle": (r"\bORA-[0-9][0-9][0-9][0-9]", r"Oracle error", r"Oracle.*Driver", r"Warning.*\Woci_.*", r"Warning.*\Wora_.*"),
+ "IBM DB2": (r"CLI Driver.*DB2", r"DB2 SQL error", r"\bdb2_\w+\("),
+ "SQLite": (r"SQLite/JDBCDriver", r"SQLite.Exception", r"System.Data.SQLite.SQLiteException", r"Warning.*sqlite_.*",
+ r"Warning.*SQLite3::", r"\[SQLITE_ERROR\]"),
+ "Sybase": (r"(?i)Warning.*sybase.*", r"Sybase message", r"Sybase.*Server message.*"),
+ }
+ for dbms, regexes in DBMS_ERRORS.items():
+ for regex in regexes:
+ if re.search(regex, new_body) and not re.search(regex, original_body):
+ return SQLiData(request_URL,
+ injection_point,
+ regex,
+ dbms)
+
+
+# A qc is either ' or "
+def inside_quote(qc: str, substring: bytes, text_index: int, body: bytes) -> bool:
+ """ Whether the Numberth occurence of the first string in the second
+ string is inside quotes as defined by the supplied QuoteChar """
+ substring = substring.decode('utf-8')
+ body = body.decode('utf-8')
+ num_substrings_found = 0
+ in_quote = False
+ for index, char in enumerate(body):
+ # Whether the next chunk of len(substring) chars is the substring
+ next_part_is_substring = (
+ (not (index + len(substring) > len(body))) and
+ (body[index:index + len(substring)] == substring)
+ )
+ # Whether this char is escaped with a \
+ is_not_escaped = (
+ (index - 1 < 0 or index - 1 > len(body)) or
+ (body[index - 1] != "\\")
+ )
+ if char == qc and is_not_escaped:
+ in_quote = not in_quote
+ if next_part_is_substring:
+ if num_substrings_found == text_index:
+ return in_quote
+ num_substrings_found += 1
+ return False
+
+
+def paths_to_text(html: str, str: str) -> List[str]:
+ """ Return list of Paths to a given str in the given HTML tree
+ - Note that it does a BFS """
+
+ def remove_last_occurence_of_sub_string(str: str, substr: str):
+ """ Delete the last occurence of substr from str
+ String String -> String
+ """
+ index = str.rfind(substr)
+ return str[:index] + str[index + len(substr):]
+
+ class PathHTMLParser(HTMLParser):
+ currentPath = ""
+ paths = []
+
+ def handle_starttag(self, tag, attrs):
+ self.currentPath += ("/" + tag)
+
+ def handle_endtag(self, tag):
+ self.currentPath = remove_last_occurence_of_sub_string(self.currentPath, "/" + tag)
+
+ def handle_data(self, data):
+ if str in data:
+ self.paths.append(self.currentPath)
+
+ parser = PathHTMLParser()
+ parser.feed(html)
+ return parser.paths
+
+
+def get_XSS_data(body: str, request_URL: str, injection_point: str) -> Optional[XSSData]:
+ """ Return a XSSDict if there is a XSS otherwise return None """
+ def in_script(text, index, body) -> bool:
+ """ Whether the Numberth occurence of the first string in the second
+ string is inside a script tag """
+ paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8"))
+ try:
+ path = paths[index]
+ return "script" in path
+ except IndexError:
+ return False
+
+ def in_HTML(text: bytes, index: int, body: bytes) -> bool:
+ """ Whether the Numberth occurence of the first string in the second
+ string is inside the HTML but not inside a script tag or part of
+ a HTML attribute"""
+ # if there is a < then lxml will interpret that as a tag, so only search for the stuff before it
+ text = text.split(b"<")[0]
+ paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8"))
+ try:
+ path = paths[index]
+ return "script" not in path
+ except IndexError:
+ return False
+
+ def inject_javascript_handler(html: str) -> bool:
+ """ Whether you can inject a Javascript:alert(0) as a link """
+ class injectJSHandlerHTMLParser(HTMLParser):
+ injectJSHandler = False
+
+ def handle_starttag(self, tag, attrs):
+ for name, value in attrs:
+ if name == "href" and value.startswith(FRONT_WALL.decode('utf-8')):
+ self.injectJSHandler = True
+
+ parser = injectJSHandlerHTMLParser()
+ parser.feed(html)
+ return parser.injectJSHandler
+ # Only convert the body to bytes if needed
+ if isinstance(body, str):
+ body = bytes(body, 'utf-8')
+ # Regex for between 24 and 72 (aka 24*3) characters encapsulated by the walls
+ regex = re.compile(b"""%s.{24,72}?%s""" % (FRONT_WALL, BACK_WALL))
+ matches = regex.findall(body)
+ for index, match in enumerate(matches):
+ # Where the string is injected into the HTML
+ in_script = in_script(match, index, body)
+ in_HTML = in_HTML(match, index, body)
+ in_tag = not in_script and not in_HTML
+ in_single_quotes = inside_quote("'", match, index, body)
+ in_double_quotes = inside_quote('"', match, index, body)
+ # Whether you can inject:
+ inject_open_angle = b"ao<ac" in match # open angle brackets
+ inject_close_angle = b"ac>so" in match # close angle brackets
+ inject_single_quotes = b"s'd" in match # single quotes
+ inject_double_quotes = b'd"ao' in match # double quotes
+ inject_slash = b"sl/bsl" in match # forward slashes
+ inject_semi = b"se;sl" in match # semicolons
+ inject_equals = b"eq=" in match # equals sign
+ if in_script and inject_slash and inject_open_angle and inject_close_angle: # e.g. <script>PAYLOAD</script>
+ return XSSData(request_URL,
+ injection_point,
+ '</script><script>alert(0)</script><script>',
+ match.decode('utf-8'))
+ elif in_script and in_single_quotes and inject_single_quotes and inject_semi: # e.g. <script>t='PAYLOAD';</script>
+ return XSSData(request_URL,
+ injection_point,
+ "';alert(0);g='",
+ match.decode('utf-8'))
+ elif in_script and in_double_quotes and inject_double_quotes and inject_semi: # e.g. <script>t="PAYLOAD";</script>
+ return XSSData(request_URL,
+ injection_point,
+ '";alert(0);g="',
+ match.decode('utf-8'))
+ elif in_tag and in_single_quotes and inject_single_quotes and inject_open_angle and inject_close_angle and inject_slash:
+ # e.g. <a href='PAYLOAD'>Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ "'><script>alert(0)</script>",
+ match.decode('utf-8'))
+ elif in_tag and in_double_quotes and inject_double_quotes and inject_open_angle and inject_close_angle and inject_slash:
+ # e.g. <a href="PAYLOAD">Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ '"><script>alert(0)</script>',
+ match.decode('utf-8'))
+ elif in_tag and not in_double_quotes and not in_single_quotes and inject_open_angle and inject_close_angle and inject_slash:
+ # e.g. <a href=PAYLOAD>Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ '><script>alert(0)</script>',
+ match.decode('utf-8'))
+ elif inject_javascript_handler(body.decode('utf-8')): # e.g. <html><a href=PAYLOAD>Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ 'Javascript:alert(0)',
+ match.decode('utf-8'))
+ elif in_tag and in_double_quotes and inject_double_quotes and inject_equals: # e.g. <a href="PAYLOAD">Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ '" onmouseover="alert(0)" t="',
+ match.decode('utf-8'))
+ elif in_tag and in_single_quotes and inject_single_quotes and inject_equals: # e.g. <a href='PAYLOAD'>Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ "' onmouseover='alert(0)' t='",
+ match.decode('utf-8'))
+ elif in_tag and not in_single_quotes and not in_double_quotes and inject_equals: # e.g. <a href=PAYLOAD>Test</a>
+ return XSSData(request_URL,
+ injection_point,
+ " onmouseover=alert(0) t=",
+ match.decode('utf-8'))
+ elif in_HTML and not in_script and inject_open_angle and inject_close_angle and inject_slash: # e.g. <html>PAYLOAD</html>
+ return XSSData(request_URL,
+ injection_point,
+ '<script>alert(0)</script>',
+ match.decode('utf-8'))
+ else:
+ return None
+
+
+# response is mitmproxy's entry point
+def response(flow: http.HTTPFlow) -> None:
+ cookiesDict = get_cookies(flow)
+ # Example: http://xss.guru/unclaimedScriptTag.html
+ find_unclaimed_URLs(flow.response.content, flow.request.url)
+ results = test_end_of_URL_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+ log_XSS_data(results[0])
+ log_SQLi_data(results[1])
+ # Example: https://daviddworken.com/vulnerableReferer.php
+ results = test_referer_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+ log_XSS_data(results[0])
+ log_SQLi_data(results[1])
+ # Example: https://daviddworken.com/vulnerableUA.php
+ results = test_user_agent_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+ log_XSS_data(results[0])
+ log_SQLi_data(results[1])
+ if "?" in flow.request.url:
+ # Example: https://daviddworken.com/vulnerable.php?name=
+ results = test_query_injection(flow.response.content.decode('utf-8'), flow.request.url, cookiesDict)
+ log_XSS_data(results[0])
+ log_SQLi_data(results[1])
diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py
index 4b939c80..6485eed7 100644
--- a/mitmproxy/certs.py
+++ b/mitmproxy/certs.py
@@ -93,9 +93,9 @@ def dummy_cert(privkey, cacert, commonname, sans):
try:
ipaddress.ip_address(i.decode("ascii"))
except ValueError:
- ss.append(b"DNS: %s" % i)
+ ss.append(b"DNS:%s" % i)
else:
- ss.append(b"IP: %s" % i)
+ ss.append(b"IP:%s" % i)
ss = b", ".join(ss)
cert = OpenSSL.crypto.X509()
@@ -356,14 +356,14 @@ class CertStore:
class _GeneralName(univ.Choice):
- # We are only interested in dNSNames. We use a default handler to ignore
- # other types.
- # TODO: We should also handle iPAddresses.
+ # We only care about dNSName and iPAddress
componentType = namedtype.NamedTypes(
namedtype.NamedType('dNSName', char.IA5String().subtype(
implicitTag=tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 2)
- )
- ),
+ )),
+ namedtype.NamedType('iPAddress', univ.OctetString().subtype(
+ implicitTag=tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 7)
+ )),
)
@@ -477,5 +477,10 @@ class SSLCert(serializable.Serializable):
except PyAsn1Error:
continue
for i in dec[0]:
- altnames.append(i[0].asOctets())
+ if i[0] is None and isinstance(i[1], univ.OctetString) and not isinstance(i[1], char.IA5String):
+ # This would give back the IP address: b'.'.join([str(e).encode() for e in i[1].asNumbers()])
+ continue
+ else:
+ e = i[0].asOctets()
+ altnames.append(e)
return altnames
diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py
index f914c7d2..9359b67d 100644
--- a/mitmproxy/connections.py
+++ b/mitmproxy/connections.py
@@ -54,24 +54,35 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
return bool(self.connection) and not self.finished
def __repr__(self):
+ if self.ssl_established:
+ tls = "[{}] ".format(self.tls_version)
+ else:
+ tls = ""
+
if self.alpn_proto_negotiated:
alpn = "[ALPN: {}] ".format(
strutils.bytes_to_escaped_str(self.alpn_proto_negotiated)
)
else:
alpn = ""
- return "<ClientConnection: {ssl}{alpn}{address}>".format(
- ssl="[ssl] " if self.ssl_established else "",
+
+ return "<ClientConnection: {tls}{alpn}{host}:{port}>".format(
+ tls=tls,
alpn=alpn,
- address=repr(self.address)
+ host=self.address[0],
+ port=self.address[1],
)
@property
def tls_established(self):
return self.ssl_established
+ @tls_established.setter
+ def tls_established(self, value):
+ self.ssl_established = value
+
_stateobject_attributes = dict(
- address=tcp.Address,
+ address=tuple,
ssl_established=bool,
clientcert=certs.SSLCert,
mitmcert=certs.SSLCert,
@@ -99,7 +110,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
@classmethod
def make_dummy(cls, address):
return cls.from_state(dict(
- address=dict(address=address, use_ipv6=False),
+ address=address,
clientcert=None,
mitmcert=None,
ssl_established=False,
@@ -143,6 +154,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
cert: The certificate presented by the remote during the TLS handshake
sni: Server Name Indication sent by the proxy during the TLS handshake
alpn_proto_negotiated: The negotiated application protocol
+ tls_version: TLS version
via: The underlying server connection (e.g. the connection to the upstream proxy in upstream proxy mode)
timestamp_start: Connection start timestamp
timestamp_tcp_setup: TCP ACK received timestamp
@@ -154,6 +166,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
tcp.TCPClient.__init__(self, address, source_address, spoof_source_address)
self.alpn_proto_negotiated = None
+ self.tls_version = None
self.via = None
self.timestamp_start = None
self.timestamp_end = None
@@ -165,35 +178,41 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
def __repr__(self):
if self.ssl_established and self.sni:
- ssl = "[ssl: {0}] ".format(self.sni)
+ tls = "[{}: {}] ".format(self.tls_version or "TLS", self.sni)
elif self.ssl_established:
- ssl = "[ssl] "
+ tls = "[{}] ".format(self.tls_version or "TLS")
else:
- ssl = ""
+ tls = ""
if self.alpn_proto_negotiated:
alpn = "[ALPN: {}] ".format(
strutils.bytes_to_escaped_str(self.alpn_proto_negotiated)
)
else:
alpn = ""
- return "<ServerConnection: {ssl}{alpn}{address}>".format(
- ssl=ssl,
+ return "<ServerConnection: {tls}{alpn}{host}:{port}>".format(
+ tls=tls,
alpn=alpn,
- address=repr(self.address)
+ host=self.address[0],
+ port=self.address[1],
)
@property
def tls_established(self):
return self.ssl_established
+ @tls_established.setter
+ def tls_established(self, value):
+ self.ssl_established = value
+
_stateobject_attributes = dict(
- address=tcp.Address,
- ip_address=tcp.Address,
- source_address=tcp.Address,
+ address=tuple,
+ ip_address=tuple,
+ source_address=tuple,
ssl_established=bool,
cert=certs.SSLCert,
sni=str,
alpn_proto_negotiated=bytes,
+ tls_version=str,
timestamp_start=float,
timestamp_tcp_setup=float,
timestamp_ssl_setup=float,
@@ -209,12 +228,13 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
@classmethod
def make_dummy(cls, address):
return cls.from_state(dict(
- address=dict(address=address, use_ipv6=False),
- ip_address=dict(address=address, use_ipv6=False),
+ address=address,
+ ip_address=address,
cert=None,
sni=None,
alpn_proto_negotiated=None,
- source_address=dict(address=('', 0), use_ipv6=False),
+ tls_version=None,
+ source_address=('', 0),
ssl_established=False,
timestamp_start=None,
timestamp_tcp_setup=None,
@@ -244,13 +264,14 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
else:
path = os.path.join(
clientcerts,
- self.address.host.encode("idna").decode()) + ".pem"
+ self.address[0].encode("idna").decode()) + ".pem"
if os.path.exists(path):
clientcert = path
self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)
self.sni = sni
self.alpn_proto_negotiated = self.get_alpn_proto_negotiated()
+ self.tls_version = self.connection.get_protocol_version_name()
self.timestamp_ssl_setup = time.time()
def finish(self):
diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py
index 4a0eeeb1..7c4f95f7 100644
--- a/mitmproxy/flowfilter.py
+++ b/mitmproxy/flowfilter.py
@@ -348,7 +348,10 @@ class FSrc(_Rex):
is_binary = False
def __call__(self, f):
- return f.client_conn.address and self.re.search(repr(f.client_conn.address))
+ if not f.client_conn or not f.client_conn.address:
+ return False
+ r = "{}:{}".format(f.client_conn.address[0], f.client_conn.address[1])
+ return f.client_conn.address and self.re.search(r)
class FDst(_Rex):
@@ -357,7 +360,10 @@ class FDst(_Rex):
is_binary = False
def __call__(self, f):
- return f.server_conn.address and self.re.search(repr(f.server_conn.address))
+ if not f.server_conn or not f.server_conn.address:
+ return False
+ r = "{}:{}".format(f.server_conn.address[0], f.server_conn.address[1])
+ return f.server_conn.address and self.re.search(r)
class _Int(_Action):
@@ -425,6 +431,7 @@ filter_unary = [
FReq,
FResp,
FTCP,
+ FWebSocket,
]
filter_rex = [
FBod,
diff --git a/mitmproxy/http.py b/mitmproxy/http.py
index f0cabcf8..c6b17533 100644
--- a/mitmproxy/http.py
+++ b/mitmproxy/http.py
@@ -5,7 +5,6 @@ from mitmproxy import flow
from mitmproxy.net import http
from mitmproxy import version
-from mitmproxy.net import tcp
from mitmproxy import connections # noqa
@@ -245,9 +244,8 @@ def make_error_response(
def make_connect_request(address):
- address = tcp.Address.wrap(address)
return HTTPRequest(
- "authority", b"CONNECT", None, address.host, address.port, None, b"HTTP/1.1",
+ "authority", b"CONNECT", None, address[0], address[1], None, b"HTTP/1.1",
http.Headers(), b""
)
diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py
index c12d2098..16cbc9fe 100644
--- a/mitmproxy/io_compat.py
+++ b/mitmproxy/io_compat.py
@@ -88,12 +88,20 @@ def convert_019_100(data):
def convert_100_200(data):
data["version"] = (2, 0, 0)
+ data["client_conn"]["address"] = data["client_conn"]["address"]["address"]
+ data["server_conn"]["address"] = data["server_conn"]["address"]["address"]
+ data["server_conn"]["source_address"] = data["server_conn"]["source_address"]["address"]
+ if data["server_conn"]["ip_address"]:
+ data["server_conn"]["ip_address"] = data["server_conn"]["ip_address"]["address"]
return data
def convert_200_300(data):
data["version"] = (3, 0, 0)
data["client_conn"]["mitmcert"] = None
+ data["server_conn"]["tls_version"] = None
+ if data["server_conn"]["via"]:
+ data["server_conn"]["via"]["tls_version"] = None
return data
diff --git a/mitmproxy/master.py b/mitmproxy/master.py
index 3a3f4399..633f32aa 100644
--- a/mitmproxy/master.py
+++ b/mitmproxy/master.py
@@ -149,8 +149,8 @@ class Master:
"""
if isinstance(f, http.HTTPFlow):
if self.server and self.options.mode == "reverse":
- f.request.host = self.server.config.upstream_server.address.host
- f.request.port = self.server.config.upstream_server.address.port
+ f.request.host = self.server.config.upstream_server.address[0]
+ f.request.port = self.server.config.upstream_server.address[1]
f.request.scheme = self.server.config.upstream_server.scheme
f.reply = controller.DummyReply()
for e, o in eventsequence.iterate(f):
diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py
index f793d397..d30c1df6 100644
--- a/mitmproxy/net/check.py
+++ b/mitmproxy/net/check.py
@@ -1,3 +1,4 @@
+import ipaddress
import re
# Allow underscore in host name
@@ -6,17 +7,26 @@ _label_valid = re.compile(b"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE)
def is_valid_host(host: bytes) -> bool:
"""
- Checks if a hostname is valid.
+ Checks if the passed bytes are a valid DNS hostname or an IPv4/IPv6 address.
"""
try:
host.decode("idna")
except ValueError:
return False
+ # RFC1035: 255 bytes or less.
if len(host) > 255:
return False
if host and host[-1:] == b".":
host = host[:-1]
- return all(_label_valid.match(x) for x in host.split(b"."))
+ # DNS hostname
+ if all(_label_valid.match(x) for x in host.split(b".")):
+ return True
+ # IPv4/IPv6 address
+ try:
+ ipaddress.ip_address(host.decode('idna'))
+ return True
+ except ValueError:
+ return False
def is_valid_port(port):
diff --git a/mitmproxy/net/socks.py b/mitmproxy/net/socks.py
index a972283e..570a4afb 100644
--- a/mitmproxy/net/socks.py
+++ b/mitmproxy/net/socks.py
@@ -2,7 +2,6 @@ import struct
import array
import ipaddress
-from mitmproxy.net import tcp
from mitmproxy.net import check
from mitmproxy.types import bidi
@@ -179,7 +178,7 @@ class Message:
self.ver = ver
self.msg = msg
self.atyp = atyp
- self.addr = tcp.Address.wrap(addr)
+ self.addr = addr
def assert_socks5(self):
if self.ver != VERSION.SOCKS5:
@@ -199,37 +198,34 @@ class Message:
if atyp == ATYP.IPV4_ADDRESS:
# We use tnoa here as ntop is not commonly available on Windows.
host = ipaddress.IPv4Address(f.safe_read(4)).compressed
- use_ipv6 = False
elif atyp == ATYP.IPV6_ADDRESS:
host = ipaddress.IPv6Address(f.safe_read(16)).compressed
- use_ipv6 = True
elif atyp == ATYP.DOMAINNAME:
length, = struct.unpack("!B", f.safe_read(1))
host = f.safe_read(length)
if not check.is_valid_host(host):
raise SocksError(REP.GENERAL_SOCKS_SERVER_FAILURE, "Invalid hostname: %s" % host)
host = host.decode("idna")
- use_ipv6 = False
else:
raise SocksError(REP.ADDRESS_TYPE_NOT_SUPPORTED,
"Socks Request: Unknown ATYP: %s" % atyp)
port, = struct.unpack("!H", f.safe_read(2))
- addr = tcp.Address((host, port), use_ipv6=use_ipv6)
+ addr = (host, port)
return cls(ver, msg, atyp, addr)
def to_file(self, f):
f.write(struct.pack("!BBBB", self.ver, self.msg, 0x00, self.atyp))
if self.atyp == ATYP.IPV4_ADDRESS:
- f.write(ipaddress.IPv4Address(self.addr.host).packed)
+ f.write(ipaddress.IPv4Address(self.addr[0]).packed)
elif self.atyp == ATYP.IPV6_ADDRESS:
- f.write(ipaddress.IPv6Address(self.addr.host).packed)
+ f.write(ipaddress.IPv6Address(self.addr[0]).packed)
elif self.atyp == ATYP.DOMAINNAME:
- f.write(struct.pack("!B", len(self.addr.host)))
- f.write(self.addr.host.encode("idna"))
+ f.write(struct.pack("!B", len(self.addr[0])))
+ f.write(self.addr[0].encode("idna"))
else:
raise SocksError(
REP.ADDRESS_TYPE_NOT_SUPPORTED,
"Unknown ATYP: %s" % self.atyp
)
- f.write(struct.pack("!H", self.addr.port))
+ f.write(struct.pack("!H", self.addr[1]))
diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py
index eabc8006..dc5e2ee2 100644
--- a/mitmproxy/net/tcp.py
+++ b/mitmproxy/net/tcp.py
@@ -19,7 +19,6 @@ from OpenSSL import SSL
from mitmproxy import certs
from mitmproxy.utils import version_check
-from mitmproxy.types import serializable
from mitmproxy import exceptions
from mitmproxy.types import basethread
@@ -29,6 +28,10 @@ version_check.check_pyopenssl_version()
socket_fileobject = socket.SocketIO
+# workaround for https://bugs.python.org/issue29515
+# Python 3.5 and 3.6 for Windows is missing a constant
+IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41)
+
EINTR = 4
HAS_ALPN = SSL._lib.Cryptography_HAS_ALPN
@@ -299,73 +302,6 @@ class Reader(_FileLike):
raise NotImplementedError("Can only peek into (pyOpenSSL) sockets")
-class Address(serializable.Serializable):
-
- """
- This class wraps an IPv4/IPv6 tuple to provide named attributes and
- ipv6 information.
- """
-
- def __init__(self, address, use_ipv6=False):
- self.address = tuple(address)
- self.use_ipv6 = use_ipv6
-
- def get_state(self):
- return {
- "address": self.address,
- "use_ipv6": self.use_ipv6
- }
-
- def set_state(self, state):
- self.address = state["address"]
- self.use_ipv6 = state["use_ipv6"]
-
- @classmethod
- def from_state(cls, state):
- return Address(**state)
-
- @classmethod
- def wrap(cls, t):
- if isinstance(t, cls):
- return t
- else:
- return cls(t)
-
- def __call__(self):
- return self.address
-
- @property
- def host(self):
- return self.address[0]
-
- @property
- def port(self):
- return self.address[1]
-
- @property
- def use_ipv6(self):
- return self.family == socket.AF_INET6
-
- @use_ipv6.setter
- def use_ipv6(self, b):
- self.family = socket.AF_INET6 if b else socket.AF_INET
-
- def __repr__(self):
- return "{}:{}".format(self.host, self.port)
-
- def __eq__(self, other):
- if not other:
- return False
- other = Address.wrap(other)
- return (self.address, self.family) == (other.address, other.family)
-
- def __ne__(self, other):
- return not self.__eq__(other)
-
- def __hash__(self):
- return hash(self.address) ^ 42 # different hash than the tuple alone.
-
-
def ssl_read_select(rlist, timeout):
"""
This is a wrapper around select.select() which also works for SSL.Connections
@@ -452,7 +388,7 @@ class _Connection:
def __init__(self, connection):
if connection:
self.connection = connection
- self.ip_address = Address(connection.getpeername())
+ self.ip_address = connection.getpeername()
self._makefile()
else:
self.connection = None
@@ -629,28 +565,6 @@ class TCPClient(_Connection):
self.sni = None
self.spoof_source_address = spoof_source_address
- @property
- def address(self):
- return self.__address
-
- @address.setter
- def address(self, address):
- if address:
- self.__address = Address.wrap(address)
- else:
- self.__address = None
-
- @property
- def source_address(self):
- return self.__source_address
-
- @source_address.setter
- def source_address(self, source_address):
- if source_address:
- self.__source_address = Address.wrap(source_address)
- else:
- self.__source_address = None
-
def close(self):
# Make sure to close the real socket, not the SSL proxy.
# OpenSSL is really good at screwing up, i.e. when trying to recv from a failed connection,
@@ -741,34 +655,57 @@ class TCPClient(_Connection):
self.rfile.set_descriptor(self.connection)
self.wfile.set_descriptor(self.connection)
- def makesocket(self):
+ def makesocket(self, family, type, proto):
# some parties (cuckoo sandbox) need to hook this
- return socket.socket(self.address.family, socket.SOCK_STREAM)
+ return socket.socket(family, type, proto)
+
+ def create_connection(self, timeout=None):
+ # Based on the official socket.create_connection implementation of Python 3.6.
+ # https://github.com/python/cpython/blob/3cc5817cfaf5663645f4ee447eaed603d2ad290a/Lib/socket.py
+
+ err = None
+ for res in socket.getaddrinfo(self.address[0], self.address[1], 0, socket.SOCK_STREAM):
+ af, socktype, proto, canonname, sa = res
+ sock = None
+ try:
+ sock = self.makesocket(af, socktype, proto)
+ if timeout:
+ sock.settimeout(timeout)
+ if self.source_address:
+ sock.bind(self.source_address)
+ if self.spoof_source_address:
+ try:
+ if not sock.getsockopt(socket.SOL_IP, socket.IP_TRANSPARENT):
+ sock.setsockopt(socket.SOL_IP, socket.IP_TRANSPARENT, 1)
+ except Exception as e:
+ # socket.IP_TRANSPARENT might not be available on every OS and Python version
+ raise exceptions.TcpException(
+ "Failed to spoof the source address: " + e.strerror
+ )
+ sock.connect(sa)
+ return sock
+
+ except socket.error as _:
+ err = _
+ if sock is not None:
+ sock.close()
+
+ if err is not None:
+ raise err
+ else:
+ raise socket.error("getaddrinfo returns an empty list")
def connect(self):
try:
- connection = self.makesocket()
-
- if self.spoof_source_address:
- try:
- # 19 is `IP_TRANSPARENT`, which is only available on Python 3.3+ on some OSes
- if not connection.getsockopt(socket.SOL_IP, 19):
- connection.setsockopt(socket.SOL_IP, 19, 1)
- except socket.error as e:
- raise exceptions.TcpException(
- "Failed to spoof the source address: " + e.strerror
- )
- if self.source_address:
- connection.bind(self.source_address())
- connection.connect(self.address())
- self.source_address = Address(connection.getsockname())
+ connection = self.create_connection()
except (socket.error, IOError) as err:
raise exceptions.TcpException(
'Error connecting to "%s": %s' %
- (self.address.host, err)
+ (self.address[0], err)
)
self.connection = connection
- self.ip_address = Address(connection.getpeername())
+ self.source_address = connection.getsockname()
+ self.ip_address = connection.getpeername()
self._makefile()
return ConnectionCloser(self)
@@ -793,7 +730,7 @@ class BaseHandler(_Connection):
def __init__(self, connection, address, server):
super().__init__(connection)
- self.address = Address.wrap(address)
+ self.address = address
self.server = server
self.clientcert = None
@@ -915,19 +852,36 @@ class TCPServer:
request_queue_size = 20
def __init__(self, address):
- self.address = Address.wrap(address)
+ self.address = address
self.__is_shut_down = threading.Event()
self.__shutdown_request = False
- self.socket = socket.socket(self.address.family, socket.SOCK_STREAM)
- self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- self.socket.bind(self.address())
- self.address = Address.wrap(self.socket.getsockname())
+
+ if self.address == 'localhost':
+ raise socket.error("Binding to 'localhost' is prohibited. Please use '::1' or '127.0.0.1' directly.")
+
+ try:
+ # First try to bind an IPv6 socket, with possible IPv4 if the OS supports it.
+ # This allows us to accept connections for ::1 and 127.0.0.1 on the same socket.
+ # Only works if self.address == ""
+ self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self.socket.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, 0)
+ self.socket.bind(self.address)
+ except socket.error:
+ self.socket = None
+
+ if not self.socket:
+ # Binding to an IPv6 socket failed, lets fall back to IPv4.
+ self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self.socket.bind(self.address)
+
+ self.address = self.socket.getsockname()
self.socket.listen(self.request_queue_size)
self.handler_counter = Counter()
def connection_thread(self, connection, client_address):
with self.handler_counter:
- client_address = Address(client_address)
try:
self.handle_client_connection(connection, client_address)
except:
@@ -954,8 +908,8 @@ class TCPServer:
self.__class__.__name__,
client_address[0],
client_address[1],
- self.address.host,
- self.address.port
+ self.address[0],
+ self.address[1],
),
target=self.connection_thread,
args=(connection, client_address),
@@ -964,7 +918,7 @@ class TCPServer:
try:
t.start()
except threading.ThreadError:
- self.handle_error(connection, Address(client_address))
+ self.handle_error(connection, client_address)
connection.close()
finally:
self.__shutdown_request = False
diff --git a/mitmproxy/net/wsgi.py b/mitmproxy/net/wsgi.py
index 8bc5bb89..a40dceca 100644
--- a/mitmproxy/net/wsgi.py
+++ b/mitmproxy/net/wsgi.py
@@ -4,14 +4,13 @@ import urllib
import io
from mitmproxy.net import http
-from mitmproxy.net import tcp
from mitmproxy.utils import strutils
class ClientConn:
def __init__(self, address):
- self.address = tcp.Address.wrap(address)
+ self.address = address
class Flow:
@@ -84,8 +83,8 @@ class WSGIAdaptor:
}
environ.update(extra)
if flow.client_conn.address:
- environ["REMOTE_ADDR"] = strutils.always_str(flow.client_conn.address.host, "latin-1")
- environ["REMOTE_PORT"] = flow.client_conn.address.port
+ environ["REMOTE_ADDR"] = strutils.always_str(flow.client_conn.address[0], "latin-1")
+ environ["REMOTE_PORT"] = flow.client_conn.address[1]
for key, value in flow.request.headers.items():
key = 'HTTP_' + strutils.always_str(key, "latin-1").upper().replace('-', '_')
diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py
index 513c0b5b..ea2f7c7f 100644
--- a/mitmproxy/proxy/config.py
+++ b/mitmproxy/proxy/config.py
@@ -23,8 +23,7 @@ class HostMatcher:
def __call__(self, address):
if not address:
return False
- address = tcp.Address.wrap(address)
- host = "%s:%s" % (address.host, address.port)
+ host = "%s:%s" % address
if any(rex.search(host) for rex in self.regexes):
return True
else:
@@ -47,7 +46,7 @@ def parse_server_spec(spec):
"Invalid server specification: %s" % spec
)
host, port = p[1:3]
- address = tcp.Address((host.decode("ascii"), port))
+ address = (host.decode("ascii"), port)
scheme = p[0].decode("ascii").lower()
return ServerSpec(scheme, address)
diff --git a/mitmproxy/proxy/protocol/base.py b/mitmproxy/proxy/protocol/base.py
index 93619171..b10bb8f5 100644
--- a/mitmproxy/proxy/protocol/base.py
+++ b/mitmproxy/proxy/protocol/base.py
@@ -101,7 +101,7 @@ class ServerConnectionMixin:
self.server_conn = None
if self.config.options.spoof_source_address and self.config.options.upstream_bind_address == '':
self.server_conn = connections.ServerConnection(
- server_address, (self.ctx.client_conn.address.host, 0), True)
+ server_address, (self.ctx.client_conn.address[0], 0), True)
else:
self.server_conn = connections.ServerConnection(
server_address, (self.config.options.upstream_bind_address, 0),
@@ -118,8 +118,8 @@ class ServerConnectionMixin:
address = self.server_conn.address
if address:
self_connect = (
- address.port == self.config.options.listen_port and
- address.host in ("localhost", "127.0.0.1", "::1")
+ address[1] == self.config.options.listen_port and
+ address[0] in ("localhost", "127.0.0.1", "::1")
)
if self_connect:
raise exceptions.ProtocolException(
@@ -133,7 +133,7 @@ class ServerConnectionMixin:
"""
if self.server_conn.connected():
self.disconnect()
- self.log("Set new server address: " + repr(address), "debug")
+ self.log("Set new server address: {}:{}".format(address[0], address[1]), "debug")
self.server_conn.address = address
self.__check_self_connect()
@@ -150,7 +150,7 @@ class ServerConnectionMixin:
self.server_conn = connections.ServerConnection(
address,
- (self.server_conn.source_address.host, 0),
+ (self.server_conn.source_address[0], 0),
self.config.options.spoof_source_address
)
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index a351ad66..b6f8463d 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -8,7 +8,6 @@ from mitmproxy import http
from mitmproxy import flow
from mitmproxy.proxy.protocol import base
from mitmproxy.proxy.protocol.websocket import WebSocketLayer
-from mitmproxy.net import tcp
from mitmproxy.net import websockets
@@ -59,7 +58,7 @@ class ConnectServerConnection:
"""
def __init__(self, address, ctx):
- self.address = tcp.Address.wrap(address)
+ self.address = address
self._ctx = ctx
@property
@@ -112,9 +111,8 @@ class UpstreamConnectLayer(base.Layer):
def set_server(self, address):
if self.ctx.server_conn.connected():
self.ctx.disconnect()
- address = tcp.Address.wrap(address)
- self.connect_request.host = address.host
- self.connect_request.port = address.port
+ self.connect_request.host = address[0]
+ self.connect_request.port = address[1]
self.server_conn.address = address
@@ -291,7 +289,7 @@ class HttpLayer(base.Layer):
# update host header in reverse proxy mode
if self.config.options.mode == "reverse" and not self.config.options.keep_host_header:
- f.request.host_header = self.config.upstream_server.address.host
+ f.request.host_header = self.config.upstream_server.address[0]
# Determine .scheme, .host and .port attributes for inline scripts. For
# absolute-form requests, they are directly given in the request. For
@@ -302,8 +300,8 @@ class HttpLayer(base.Layer):
# Setting request.host also updates the host header, which we want
# to preserve
host_header = f.request.host_header
- f.request.host = self.__initial_server_conn.address.host
- f.request.port = self.__initial_server_conn.address.port
+ f.request.host = self.__initial_server_conn.address[0]
+ f.request.port = self.__initial_server_conn.address[1]
f.request.host_header = host_header # set again as .host overwrites this.
f.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", f)
@@ -453,14 +451,14 @@ class HttpLayer(base.Layer):
self.set_server(address)
def establish_server_connection(self, host: str, port: int, scheme: str):
- address = tcp.Address((host, port))
tls = (scheme == "https")
if self.mode is HTTPMode.regular or self.mode is HTTPMode.transparent:
# If there's an existing connection that doesn't match our expectations, kill it.
+ address = (host, port)
if address != self.server_conn.address or tls != self.server_tls:
self.set_server(address)
- self.set_server_tls(tls, address.host)
+ self.set_server_tls(tls, address[0])
# Establish connection is neccessary.
if not self.server_conn.connected():
self.connect()
diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py
index cdce24b3..01406798 100644
--- a/mitmproxy/proxy/protocol/http2.py
+++ b/mitmproxy/proxy/protocol/http2.py
@@ -97,7 +97,6 @@ class Http2Layer(base.Layer):
client_side=False,
header_encoding=False,
validate_outbound_headers=False,
- normalize_outbound_headers=False,
validate_inbound_headers=False)
self.connections[self.client_conn] = SafeH2Connection(self.client_conn, config=config)
@@ -107,7 +106,6 @@ class Http2Layer(base.Layer):
client_side=True,
header_encoding=False,
validate_outbound_headers=False,
- normalize_outbound_headers=False,
validate_inbound_headers=False)
self.connections[self.server_conn] = SafeH2Connection(self.server_conn, config=config)
self.connections[self.server_conn].initiate_connection()
@@ -599,9 +597,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr
def send_response_headers(self, response):
headers = response.headers.copy()
headers.insert(0, ":status", str(response.status_code))
- for forbidden_header in h2.utilities.CONNECTION_HEADERS:
- if forbidden_header in headers:
- del headers[forbidden_header]
with self.connections[self.client_conn].lock:
self.connections[self.client_conn].safe_send_headers(
self.raise_zombie,
diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py
index 08ce53d0..7d15130f 100644
--- a/mitmproxy/proxy/protocol/tls.py
+++ b/mitmproxy/proxy/protocol/tls.py
@@ -545,8 +545,9 @@ class TlsLayer(base.Layer):
raise exceptions.InvalidServerCertificate(str(e))
except exceptions.TlsException as e:
raise exceptions.TlsProtocolException(
- "Cannot establish TLS with {address} (sni: {sni}): {e}".format(
- address=repr(self.server_conn.address),
+ "Cannot establish TLS with {host}:{port} (sni: {sni}): {e}".format(
+ host=self.server_conn.address[0],
+ port=self.server_conn.address[1],
sni=self.server_sni,
e=repr(e)
)
@@ -567,7 +568,7 @@ class TlsLayer(base.Layer):
# However, we may just want to establish TLS so that we can send an error message to the client,
# in which case the address can be None.
if self.server_conn.address:
- host = self.server_conn.address.host.encode("idna")
+ host = self.server_conn.address[0].encode("idna")
# Should we incorporate information from the server certificate?
use_upstream_cert = (
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index 180fc9ca..1987c8dc 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -68,7 +68,7 @@ class RootContext:
top_layer,
client_tls,
top_layer.server_tls,
- top_layer.server_conn.address.host
+ top_layer.server_conn.address[0]
)
if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer):
return protocol.TlsLayer(top_layer, client_tls, client_tls)
@@ -104,7 +104,7 @@ class RootContext:
Send a log message to the master.
"""
full_msg = [
- "{}: {}".format(repr(self.client_conn.address), msg)
+ "{}:{}: {}".format(self.client_conn.address[0], self.client_conn.address[1], msg)
]
for i in subs:
full_msg.append(" -> " + i)
diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py
index 97018dad..8082cb64 100644
--- a/mitmproxy/proxy/server.py
+++ b/mitmproxy/proxy/server.py
@@ -1,4 +1,3 @@
-import socket
import sys
import traceback
@@ -46,10 +45,10 @@ class ProxyServer(tcp.TCPServer):
)
if config.options.mode == "transparent":
platform.init_transparent_mode()
- except socket.error as e:
+ except Exception as e:
raise exceptions.ServerException(
'Error starting proxy server: ' + repr(e)
- )
+ ) from e
self.channel = None
def set_channel(self, channel):
diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py
index ea7be4b9..fd665055 100644
--- a/mitmproxy/test/tflow.py
+++ b/mitmproxy/test/tflow.py
@@ -1,3 +1,5 @@
+import io
+
from mitmproxy.net import websockets
from mitmproxy.test import tutils
from mitmproxy import tcp
@@ -72,7 +74,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None,
if messages is True:
messages = [
websocket.WebSocketMessage(websockets.OPCODE.BINARY, True, b"hello binary"),
- websocket.WebSocketMessage(websockets.OPCODE.TEXT, False, "hello text".encode()),
+ websocket.WebSocketMessage(websockets.OPCODE.TEXT, True, "hello text".encode()),
+ websocket.WebSocketMessage(websockets.OPCODE.TEXT, False, "it's me".encode()),
]
if err is True:
err = terr()
@@ -142,7 +145,7 @@ def tclient_conn():
@return: mitmproxy.proxy.connection.ClientConnection
"""
c = connections.ClientConnection.from_state(dict(
- address=dict(address=("address", 22), use_ipv6=True),
+ address=("address", 22),
clientcert=None,
mitmcert=None,
ssl_established=False,
@@ -155,6 +158,8 @@ def tclient_conn():
tls_version="TLSv1.2",
))
c.reply = controller.DummyReply()
+ c.rfile = io.BytesIO()
+ c.wfile = io.BytesIO()
return c
@@ -163,8 +168,8 @@ def tserver_conn():
@return: mitmproxy.proxy.connection.ServerConnection
"""
c = connections.ServerConnection.from_state(dict(
- address=dict(address=("address", 22), use_ipv6=True),
- source_address=dict(address=("address", 22), use_ipv6=True),
+ address=("address", 22),
+ source_address=("address", 22),
ip_address=None,
cert=None,
timestamp_start=1,
@@ -174,9 +179,12 @@ def tserver_conn():
ssl_established=False,
sni="address",
alpn_proto_negotiated=None,
+ tls_version=None,
via=None,
))
c.reply = controller.DummyReply()
+ c.rfile = io.BytesIO()
+ c.wfile = io.BytesIO()
return c
diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py
index d713787a..691f19a5 100644
--- a/mitmproxy/tools/console/flowdetailview.py
+++ b/mitmproxy/tools/console/flowdetailview.py
@@ -30,8 +30,8 @@ def flowdetails(state, flow: http.HTTPFlow):
if sc is not None:
text.append(urwid.Text([("head", "Server Connection:")]))
parts = [
- ["Address", repr(sc.address)],
- ["Resolved Address", repr(sc.ip_address)],
+ ["Address", "{}:{}".format(sc.address[0], sc.address[1])],
+ ["Resolved Address", "{}:{}".format(sc.ip_address[0], sc.ip_address[1])],
]
if resp:
parts.append(["HTTP Version", resp.http_version])
@@ -92,7 +92,7 @@ def flowdetails(state, flow: http.HTTPFlow):
text.append(urwid.Text([("head", "Client Connection:")]))
parts = [
- ["Address", repr(cc.address)],
+ ["Address", "{}:{}".format(cc.address[0], cc.address[1])],
]
if req:
parts.append(["HTTP Version", req.http_version])
diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py
index a97a9b31..90cca1c5 100644
--- a/mitmproxy/tools/console/flowview.py
+++ b/mitmproxy/tools/console/flowview.py
@@ -681,7 +681,7 @@ class FlowView(tabs.Tabs):
encoding_map = {
"z": "gzip",
"d": "deflate",
- "b": "brotli",
+ "b": "br",
}
conn.encode(encoding_map[key])
signals.flow_change.send(self, flow = self.flow)
diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py
index 4ab9e1f4..d68dc93c 100644
--- a/mitmproxy/tools/console/master.py
+++ b/mitmproxy/tools/console/master.py
@@ -429,9 +429,11 @@ class ConsoleMaster(master.Master):
super().tcp_message(f)
message = f.messages[-1]
direction = "->" if message.from_client else "<-"
- signals.add_log("{client} {direction} tcp {direction} {server}".format(
- client=repr(f.client_conn.address),
- server=repr(f.server_conn.address),
+ signals.add_log("{client_host}:{client_port} {direction} tcp {direction} {server_host}:{server_port}".format(
+ client_host=f.client_conn.address[0],
+ client_port=f.client_conn.address[1],
+ server_host=f.server_conn.address[0],
+ server_port=f.server_conn.address[1],
direction=direction,
), "info")
signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug")
diff --git a/mitmproxy/tools/console/palettepicker.py b/mitmproxy/tools/console/palettepicker.py
index 4c5c62a0..1f238b0d 100644
--- a/mitmproxy/tools/console/palettepicker.py
+++ b/mitmproxy/tools/console/palettepicker.py
@@ -43,7 +43,7 @@ class PalettePicker(urwid.WidgetWrap):
i,
None,
lambda: self.master.options.console_palette == name,
- lambda: setattr(self.master.options, "palette", name)
+ lambda: setattr(self.master.options, "console_palette", name)
)
for i in high:
@@ -59,7 +59,7 @@ class PalettePicker(urwid.WidgetWrap):
"Transparent",
"T",
lambda: master.options.console_palette_transparent,
- master.options.toggler("palette_transparent")
+ master.options.toggler("console_palette_transparent")
)
]
)
diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py
index 2c7f9efb..d90d932b 100644
--- a/mitmproxy/tools/console/statusbar.py
+++ b/mitmproxy/tools/console/statusbar.py
@@ -238,8 +238,8 @@ class StatusBar(urwid.WidgetWrap):
dst = self.master.server.config.upstream_server
r.append("[dest:%s]" % mitmproxy.net.http.url.unparse(
dst.scheme,
- dst.address.host,
- dst.address.port
+ dst.address[0],
+ dst.address[1],
))
if self.master.options.scripts:
r.append("[")
@@ -272,10 +272,10 @@ class StatusBar(urwid.WidgetWrap):
]
if self.master.server.bound:
- host = self.master.server.address.host
+ host = self.master.server.address[0]
if host == "0.0.0.0":
host = "*"
- boundaddr = "[%s:%s]" % (host, self.master.server.address.port)
+ boundaddr = "[%s:%s]" % (host, self.master.server.address[1])
else:
boundaddr = ""
t.extend(self.get_status())
diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py
index e1e40fb0..fefbddfb 100644
--- a/mitmproxy/tools/dump.py
+++ b/mitmproxy/tools/dump.py
@@ -46,7 +46,7 @@ class DumpMaster(master.Master):
if not self.options.no_server:
self.add_log(
- "Proxy server listening at http://{}".format(server.address),
+ "Proxy server listening at http://{}:{}".format(server.address[0], server.address[1]),
"info"
)
diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py
index 1f3467cc..893c3dde 100644
--- a/mitmproxy/tools/web/app.py
+++ b/mitmproxy/tools/web/app.py
@@ -85,6 +85,7 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
"is_replay": flow.response.is_replay,
}
f.get("server_conn", {}).pop("cert", None)
+ f.get("client_conn", {}).pop("mitmcert", None)
return f
diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py
index 6ebcfe47..8c7f579d 100644
--- a/mitmproxy/tools/web/master.py
+++ b/mitmproxy/tools/web/master.py
@@ -109,7 +109,7 @@ class WebMaster(master.Master):
tornado.ioloop.PeriodicCallback(lambda: self.tick(timeout=0), 5).start()
self.add_log(
- "Proxy server listening at http://{}/".format(self.server.address),
+ "Proxy server listening at http://{}:{}/".format(self.server.address[0], self.server.address[1]),
"info"
)
diff --git a/mitmproxy/tools/web/static/app.js b/mitmproxy/tools/web/static/app.js
index 480d9c71..c21a478b 100644
--- a/mitmproxy/tools/web/static/app.js
+++ b/mitmproxy/tools/web/static/app.js
@@ -2046,7 +2046,7 @@ function ConnectionInfo(_ref2) {
_react2.default.createElement(
'td',
null,
- conn.address.address.join(':')
+ conn.address.join(':')
)
),
conn.sni && _react2.default.createElement(
@@ -8449,7 +8449,7 @@ module.exports = function () {
function destination(regex) {
regex = new RegExp(regex, "i");
function destinationFilter(flow) {
- return !!flow.server_conn.address && regex.test(flow.server_conn.address.address[0] + ":" + flow.server_conn.address.address[1]);
+ return !!flow.server_conn.address && regex.test(flow.server_conn.address[0] + ":" + flow.server_conn.address[1]);
}
destinationFilter.desc = "destination address matches " + regex;
return destinationFilter;
@@ -8509,7 +8509,7 @@ module.exports = function () {
function source(regex) {
regex = new RegExp(regex, "i");
function sourceFilter(flow) {
- return !!flow.client_conn.address && regex.test(flow.client_conn.address.address[0] + ":" + flow.client_conn.address.address[1]);
+ return !!flow.client_conn.address && regex.test(flow.client_conn.address[0] + ":" + flow.client_conn.address[1]);
}
sourceFilter.desc = "source address matches " + regex;
return sourceFilter;
diff --git a/pathod/pathoc.py b/pathod/pathoc.py
index 549444ca..4a613349 100644
--- a/pathod/pathoc.py
+++ b/pathod/pathoc.py
@@ -239,7 +239,7 @@ class Pathoc(tcp.TCPClient):
is_client=True,
staticdir=os.getcwd(),
unconstrained_file_access=True,
- request_host=self.address.host,
+ request_host=self.address[0],
protocol=self.protocol,
)
@@ -286,7 +286,7 @@ class Pathoc(tcp.TCPClient):
socks.VERSION.SOCKS5,
socks.CMD.CONNECT,
socks.ATYP.DOMAINNAME,
- tcp.Address.wrap(connect_to)
+ connect_to,
)
connect_request.to_file(self.wfile)
self.wfile.flush()
diff --git a/pathod/pathod.py b/pathod/pathod.py
index 8d57897b..7416d325 100644
--- a/pathod/pathod.py
+++ b/pathod/pathod.py
@@ -166,7 +166,7 @@ class PathodHandler(tcp.BaseHandler):
headers=headers.fields,
http_version=http_version,
sni=self.sni,
- remote_address=self.address(),
+ remote_address=self.address,
clientcert=clientcert,
first_line_format=first_line_format
),
diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py
index 628e3f33..7c88c5c7 100644
--- a/pathod/protocols/http2.py
+++ b/pathod/protocols/http2.py
@@ -172,9 +172,9 @@ class HTTP2StateProtocol:
def assemble_request(self, request):
assert isinstance(request, mitmproxy.net.http.request.Request)
- authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host
- if self.tcp_handler.address.port != 443:
- authority += ":%d" % self.tcp_handler.address.port
+ authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address[0]
+ if self.tcp_handler.address[1] != 443:
+ authority += ":%d" % self.tcp_handler.address[1]
headers = request.headers.copy()
diff --git a/pathod/test.py b/pathod/test.py
index b819d723..81f5805f 100644
--- a/pathod/test.py
+++ b/pathod/test.py
@@ -97,8 +97,8 @@ class _PaThread(basethread.BaseThread):
**self.daemonargs
)
self.name = "PathodThread (%s:%s)" % (
- self.server.address.host,
- self.server.address.port
+ self.server.address[0],
+ self.server.address[1],
)
- self.q.put(self.server.address.port)
+ self.q.put(self.server.address[1])
self.server.serve_forever()
diff --git a/setup.cfg b/setup.cfg
index 52f51f26..7fbb7f73 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -34,16 +34,11 @@ exclude =
mitmproxy/proxy/root_context.py
mitmproxy/proxy/server.py
mitmproxy/tools/
- mitmproxy/certs.py
- mitmproxy/connections.py
mitmproxy/controller.py
mitmproxy/export.py
mitmproxy/flow.py
- mitmproxy/flowfilter.py
- mitmproxy/http.py
mitmproxy/io_compat.py
mitmproxy/master.py
- mitmproxy/optmanager.py
pathod/pathoc.py
pathod/pathod.py
pathod/test.py
@@ -54,8 +49,6 @@ exclude =
mitmproxy/addonmanager.py
mitmproxy/addons/onboardingapp/app.py
mitmproxy/addons/termlog.py
- mitmproxy/certs.py
- mitmproxy/connections.py
mitmproxy/contentviews/base.py
mitmproxy/contentviews/wbxml.py
mitmproxy/contentviews/xml_html.py
@@ -64,8 +57,6 @@ exclude =
mitmproxy/exceptions.py
mitmproxy/export.py
mitmproxy/flow.py
- mitmproxy/flowfilter.py
- mitmproxy/http.py
mitmproxy/io.py
mitmproxy/io_compat.py
mitmproxy/log.py
@@ -78,7 +69,6 @@ exclude =
mitmproxy/net/http/url.py
mitmproxy/net/tcp.py
mitmproxy/options.py
- mitmproxy/optmanager.py
mitmproxy/proxy/config.py
mitmproxy/proxy/modes/http_proxy.py
mitmproxy/proxy/modes/reverse_proxy.py
diff --git a/setup.py b/setup.py
index a942a9e3..bbbac95e 100644
--- a/setup.py
+++ b/setup.py
@@ -113,7 +113,6 @@ setup(
],
'examples': [
"beautifulsoup4>=4.4.1, <4.6",
- "pytz>=2015.07.0, <=2016.10",
"Pillow>=3.2, <4.1",
]
}
diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py
index 6a66d0c9..22d2c2c6 100644
--- a/test/mitmproxy/addons/test_dumper.py
+++ b/test/mitmproxy/addons/test_dumper.py
@@ -70,7 +70,7 @@ def test_simple():
flow.request = tutils.treq()
flow.request.stickycookie = True
flow.client_conn = mock.MagicMock()
- flow.client_conn.address.host = "foo"
+ flow.client_conn.address[0] = "foo"
flow.response = tutils.tresp(content=None)
flow.response.is_replay = True
flow.response.status_code = 300
@@ -176,7 +176,7 @@ def test_websocket():
ctx.configure(d, flow_detail=3, showhost=True)
f = tflow.twebsocketflow()
d.websocket_message(f)
- assert "hello text" in sio.getvalue()
+ assert "it's me" in sio.getvalue()
sio.truncate(0)
d.websocket_end(f)
diff --git a/test/mitmproxy/examples/test_xss_scanner.py b/test/mitmproxy/examples/test_xss_scanner.py
new file mode 100755
index 00000000..14ee6902
--- /dev/null
+++ b/test/mitmproxy/examples/test_xss_scanner.py
@@ -0,0 +1,368 @@
+import pytest
+import requests
+from examples.complex import xss_scanner as xss
+from mitmproxy.test import tflow, tutils
+
+
+class TestXSSScanner():
+ def test_get_XSS_info(self):
+ # First type of exploit: <script>PAYLOAD</script>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><script>%s</script><html>" %
+ xss.FULL_PAYLOAD,
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData('https://example.com',
+ "End of URL",
+ '</script><script>alert(0)</script><script>',
+ xss.FULL_PAYLOAD.decode('utf-8'))
+ assert xss_info == expected_xss_info
+ xss_info = xss.get_XSS_data(b"<html><script>%s</script><html>" %
+ xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ '</script><script>alert(0)</script><script>',
+ xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><script>%s</script><html>" %
+ xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22").replace(b"/", b"%2F"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Second type of exploit: <script>t='PAYLOAD'</script>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><script>t='%s';</script></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").replace(b"\"", b"%22"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "';alert(0);g='",
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b"\"", b"%22").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><script>t='%s';</script></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b"\"", b"%22").replace(b"'", b"%22"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Third type of exploit: <script>t="PAYLOAD"</script>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><script>t=\"%s\";</script></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").replace(b"'", b"%27"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ '";alert(0);g="',
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b"'", b"%27").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><script>t=\"%s\";</script></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b"'", b"%27").replace(b"\"", b"%22"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Fourth type of exploit: <a href='PAYLOAD'>Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href='%s'>Test</a></html>" %
+ xss.FULL_PAYLOAD,
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "'><script>alert(0)</script>",
+ xss.FULL_PAYLOAD.decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href='OtherStuff%s'>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"'", b"%27"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Fifth type of exploit: <a href="PAYLOAD">Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=\"%s\">Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"'", b"%27"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "\"><script>alert(0)</script>",
+ xss.FULL_PAYLOAD.replace(b"'", b"%27").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=\"OtherStuff%s\">Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b"\"", b"%22"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Sixth type of exploit: <a href=PAYLOAD>Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=%s>Test</a></html>" %
+ xss.FULL_PAYLOAD,
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "><script>alert(0)</script>",
+ xss.FULL_PAYLOAD.decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable
+ xss_info = xss.get_XSS_data(b"<html><a href=OtherStuff%s>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b"=", b"%3D"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Seventh type of exploit: <html>PAYLOAD</html>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><b>%s</b></html>" %
+ xss.FULL_PAYLOAD,
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "<script>alert(0)</script>",
+ xss.FULL_PAYLOAD.decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable
+ xss_info = xss.get_XSS_data(b"<html><b>%s</b></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").replace(b"/", b"%2F"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Eighth type of exploit: <a href=PAYLOAD>Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=%s>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "Javascript:alert(0)",
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=OtherStuff%s>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b"=", b"%3D"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Ninth type of exploit: <a href="STUFF PAYLOAD">Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=\"STUFF %s\">Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ '" onmouseover="alert(0)" t="',
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=\"STUFF %s\">Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b'"', b"%22"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Tenth type of exploit: <a href='STUFF PAYLOAD'>Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href='STUFF %s'>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ "' onmouseover='alert(0)' t='",
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href='STUFF %s'>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b"'", b"%22"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+ # Eleventh type of exploit: <a href=STUFF_PAYLOAD>Test</a>
+ # Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=STUFF%s>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"),
+ "https://example.com",
+ "End of URL")
+ expected_xss_info = xss.XSSData("https://example.com",
+ "End of URL",
+ " onmouseover=alert(0) t=",
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8'))
+ assert xss_info == expected_xss_info
+ # Non-Exploitable:
+ xss_info = xss.get_XSS_data(b"<html><a href=STUFF_%s>Test</a></html>" %
+ xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E")
+ .replace(b"=", b"%3D"),
+ "https://example.com",
+ "End of URL")
+ assert xss_info is None
+
+ def test_get_SQLi_data(self):
+ sqli_data = xss.get_SQLi_data("<html>SQL syntax MySQL</html>",
+ "<html></html>",
+ "https://example.com",
+ "End of URL")
+ expected_sqli_data = xss.SQLiData("https://example.com",
+ "End of URL",
+ "SQL syntax.*MySQL",
+ "MySQL")
+ assert sqli_data == expected_sqli_data
+ sqli_data = xss.get_SQLi_data("<html>SQL syntax MySQL</html>",
+ "<html>SQL syntax MySQL</html>",
+ "https://example.com",
+ "End of URL")
+ assert sqli_data is None
+
+ def test_inside_quote(self):
+ assert not xss.inside_quote("'", b"no", 0, b"no")
+ assert xss.inside_quote("'", b"yes", 0, b"'yes'")
+ assert xss.inside_quote("'", b"yes", 1, b"'yes'otherJunk'yes'more")
+ assert not xss.inside_quote("'", b"longStringNotInIt", 1, b"short")
+
+ def test_paths_to_text(self):
+ text = xss.paths_to_text("""<html><head><h1>STRING</h1></head>
+ <script>STRING</script>
+ <a href=STRING></a></html>""", "STRING")
+ expected_text = ["/html/head/h1", "/html/script"]
+ assert text == expected_text
+ assert xss.paths_to_text("""<html></html>""", "STRING") == []
+
+ def mocked_requests_vuln(*args, headers=None, cookies=None):
+ class MockResponse:
+ def __init__(self, html, headers=None, cookies=None):
+ self.text = html
+ return MockResponse("<html>%s</html>" % xss.FULL_PAYLOAD)
+
+ def mocked_requests_invuln(*args, headers=None, cookies=None):
+ class MockResponse:
+ def __init__(self, html, headers=None, cookies=None):
+ self.text = html
+ return MockResponse("<html></html>")
+
+ def test_test_end_of_url_injection(self, monkeypatch):
+ monkeypatch.setattr(requests, 'get', self.mocked_requests_vuln)
+ xss_info = xss.test_end_of_URL_injection("<html></html>", "https://example.com/index.html", {})[0]
+ expected_xss_info = xss.XSSData('https://example.com/index.html/1029zxcs\'d"ao<ac>so[sb]po(pc)se;sl/bsl\\eq=3847asd',
+ 'End of URL',
+ '<script>alert(0)</script>',
+ '1029zxcs\\\'d"ao<ac>so[sb]po(pc)se;sl/bsl\\\\eq=3847asd')
+ sqli_info = xss.test_end_of_URL_injection("<html></html>", "https://example.com/", {})[1]
+ assert xss_info == expected_xss_info
+ assert sqli_info is None
+
+ def test_test_referer_injection(self, monkeypatch):
+ monkeypatch.setattr(requests, 'get', self.mocked_requests_vuln)
+ xss_info = xss.test_referer_injection("<html></html>", "https://example.com/", {})[0]
+ expected_xss_info = xss.XSSData('https://example.com/',
+ 'Referer',
+ '<script>alert(0)</script>',
+ '1029zxcs\\\'d"ao<ac>so[sb]po(pc)se;sl/bsl\\\\eq=3847asd')
+ sqli_info = xss.test_referer_injection("<html></html>", "https://example.com/", {})[1]
+ assert xss_info == expected_xss_info
+ assert sqli_info is None
+
+ def test_test_user_agent_injection(self, monkeypatch):
+ monkeypatch.setattr(requests, 'get', self.mocked_requests_vuln)
+ xss_info = xss.test_user_agent_injection("<html></html>", "https://example.com/", {})[0]
+ expected_xss_info = xss.XSSData('https://example.com/',
+ 'User Agent',
+ '<script>alert(0)</script>',
+ '1029zxcs\\\'d"ao<ac>so[sb]po(pc)se;sl/bsl\\\\eq=3847asd')
+ sqli_info = xss.test_user_agent_injection("<html></html>", "https://example.com/", {})[1]
+ assert xss_info == expected_xss_info
+ assert sqli_info is None
+
+ def test_test_query_injection(self, monkeypatch):
+ monkeypatch.setattr(requests, 'get', self.mocked_requests_vuln)
+ xss_info = xss.test_query_injection("<html></html>", "https://example.com/vuln.php?cmd=ls", {})[0]
+ expected_xss_info = xss.XSSData('https://example.com/vuln.php?cmd=1029zxcs\'d"ao<ac>so[sb]po(pc)se;sl/bsl\\eq=3847asd',
+ 'Query',
+ '<script>alert(0)</script>',
+ '1029zxcs\\\'d"ao<ac>so[sb]po(pc)se;sl/bsl\\\\eq=3847asd')
+ sqli_info = xss.test_query_injection("<html></html>", "https://example.com/vuln.php?cmd=ls", {})[1]
+ assert xss_info == expected_xss_info
+ assert sqli_info is None
+
+ @pytest.fixture
+ def logger(self):
+ class Logger():
+ def __init__(self):
+ self.args = []
+
+ def error(self, str):
+ self.args.append(str)
+ return Logger()
+
+ def test_find_unclaimed_URLs(self, monkeypatch, logger):
+ logger.args = []
+ monkeypatch.setattr("mitmproxy.ctx.log", logger)
+ xss.find_unclaimed_URLs("<html><script src=\"http://google.com\"></script></html>",
+ "https://example.com")
+ assert logger.args == []
+ xss.find_unclaimed_URLs("<html><script src=\"http://unclaimedDomainName.com\"></script></html>",
+ "https://example.com")
+ assert logger.args[0] == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com" in script tag.'
+
+ def test_log_XSS_data(self, monkeypatch, logger):
+ logger.args = []
+ monkeypatch.setattr("mitmproxy.ctx.log", logger)
+ xss.log_XSS_data(None)
+ assert logger.args == []
+ # self, url: str, injection_point: str, exploit: str, line: str
+ xss.log_XSS_data(xss.XSSData('https://example.com',
+ 'Location',
+ 'String',
+ 'Line of HTML'))
+ assert logger.args[0] == '===== XSS Found ===='
+ assert logger.args[1] == 'XSS URL: https://example.com'
+ assert logger.args[2] == 'Injection Point: Location'
+ assert logger.args[3] == 'Suggested Exploit: String'
+ assert logger.args[4] == 'Line: Line of HTML'
+
+ def test_log_SQLi_data(self, monkeypatch, logger):
+ logger.args = []
+ monkeypatch.setattr("mitmproxy.ctx.log", logger)
+ xss.log_SQLi_data(None)
+ assert logger.args == []
+ xss.log_SQLi_data(xss.SQLiData(b'https://example.com',
+ b'Location',
+ b'Oracle.*Driver',
+ b'Oracle'))
+ assert logger.args[0] == '===== SQLi Found ====='
+ assert logger.args[1] == 'SQLi URL: https://example.com'
+ assert logger.args[2] == 'Injection Point: Location'
+ assert logger.args[3] == 'Regex used: Oracle.*Driver'
+
+ def test_get_cookies(self):
+ mocked_req = tutils.treq()
+ mocked_req.cookies = [("cookieName2", "cookieValue2")]
+ mocked_flow = tflow.tflow(req=mocked_req)
+ # It only uses the request cookies
+ assert xss.get_cookies(mocked_flow) == {"cookieName2": "cookieValue2"}
+
+ def test_response(self, monkeypatch, logger):
+ logger.args = []
+ monkeypatch.setattr("mitmproxy.ctx.log", logger)
+ monkeypatch.setattr(requests, 'get', self.mocked_requests_invuln)
+ mocked_flow = tflow.tflow(req=tutils.treq(path=b"index.html?q=1"), resp=tutils.tresp(content=b'<html></html>'))
+ xss.response(mocked_flow)
+ assert logger.args == []
+
+ def test_data_equals(self):
+ xssData = xss.XSSData("a", "b", "c", "d")
+ sqliData = xss.SQLiData("a", "b", "c", "d")
+ assert xssData == xssData
+ assert sqliData == sqliData
diff --git a/test/mitmproxy/net/test_check.py b/test/mitmproxy/net/test_check.py
index 9dbc02e0..0ffd6b2e 100644
--- a/test/mitmproxy/net/test_check.py
+++ b/test/mitmproxy/net/test_check.py
@@ -11,3 +11,4 @@ def test_is_valid_host():
assert check.is_valid_host(b"one.two.")
# Allow underscore
assert check.is_valid_host(b"one_two")
+ assert check.is_valid_host(b"::1")
diff --git a/test/mitmproxy/net/test_socks.py b/test/mitmproxy/net/test_socks.py
index e00dd410..fbd31ef4 100644
--- a/test/mitmproxy/net/test_socks.py
+++ b/test/mitmproxy/net/test_socks.py
@@ -3,7 +3,6 @@ from io import BytesIO
import pytest
from mitmproxy.net import socks
-from mitmproxy.net import tcp
from mitmproxy.test import tutils
@@ -176,7 +175,7 @@ def test_message_ipv6():
msg.to_file(out)
assert out.getvalue() == raw.getvalue()[:-2]
- assert msg.addr.host == ipv6_addr
+ assert msg.addr[0] == ipv6_addr
def test_message_invalid_host():
@@ -196,6 +195,6 @@ def test_message_unknown_atyp():
with pytest.raises(socks.SocksError):
socks.Message.from_file(raw)
- m = socks.Message(5, 1, 0x02, tcp.Address(("example.com", 5050)))
+ m = socks.Message(5, 1, 0x02, ("example.com", 5050))
with pytest.raises(socks.SocksError):
m.to_file(BytesIO())
diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py
index ff6362c8..cf010f6e 100644
--- a/test/mitmproxy/net/test_tcp.py
+++ b/test/mitmproxy/net/test_tcp.py
@@ -116,11 +116,11 @@ class TestServerBind(tservers.ServerTestBase):
class TestServerIPv6(tservers.ServerTestBase):
handler = EchoHandler
- addr = tcp.Address(("localhost", 0), use_ipv6=True)
+ addr = ("::1", 0)
def test_echo(self):
testval = b"echo!\n"
- c = tcp.TCPClient(tcp.Address(("::1", self.port), use_ipv6=True))
+ c = tcp.TCPClient(("::1", self.port))
with c.connect():
c.wfile.write(testval)
c.wfile.flush()
@@ -132,7 +132,7 @@ class TestEcho(tservers.ServerTestBase):
def test_echo(self):
testval = b"echo!\n"
- c = tcp.TCPClient(("127.0.0.1", self.port))
+ c = tcp.TCPClient(("localhost", self.port))
with c.connect():
c.wfile.write(testval)
c.wfile.flush()
@@ -602,12 +602,6 @@ class TestDHParams(tservers.ServerTestBase):
ret = c.get_current_cipher()
assert ret[0] == "DHE-RSA-AES256-SHA"
- def test_create_dhparams(self):
- with tutils.tmpdir() as d:
- filename = os.path.join(d, "dhparam.pem")
- certs.CertStore.load_dhparam(filename)
- assert os.path.exists(filename)
-
class TestTCPClient:
@@ -783,18 +777,6 @@ class TestPeekSSL(TestPeek):
return conn.pop()
-class TestAddress:
- def test_simple(self):
- a = tcp.Address(("localhost", 80), True)
- assert a.use_ipv6
- b = tcp.Address(("foo.com", 80), True)
- assert not a == b
- c = tcp.Address(("localhost", 80), True)
- assert a == c
- assert not a != c
- assert repr(a) == "localhost:80"
-
-
class TestSSLKeyLogger(tservers.ServerTestBase):
handler = EchoHandler
ssl = dict(
diff --git a/test/mitmproxy/net/tservers.py b/test/mitmproxy/net/tservers.py
index 68a2caa0..ebe6d3eb 100644
--- a/test/mitmproxy/net/tservers.py
+++ b/test/mitmproxy/net/tservers.py
@@ -86,13 +86,13 @@ class _TServer(tcp.TCPServer):
class ServerTestBase:
ssl = None
handler = None
- addr = ("localhost", 0)
+ addr = ("127.0.0.1", 0)
@classmethod
def setup_class(cls, **kwargs):
cls.q = queue.Queue()
s = cls.makeserver(**kwargs)
- cls.port = s.address.port
+ cls.port = s.address[1]
cls.server = _ServerThread(s)
cls.server.start()
diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py
index cb9c0474..871d02fe 100644
--- a/test/mitmproxy/proxy/protocol/test_http2.py
+++ b/test/mitmproxy/proxy/protocol/test_http2.py
@@ -124,10 +124,10 @@ class _Http2TestBase:
b'CONNECT',
b'',
b'localhost',
- self.server.server.address.port,
+ self.server.server.address[1],
b'/',
b'HTTP/1.1',
- [(b'host', b'localhost:%d' % self.server.server.address.port)],
+ [(b'host', b'localhost:%d' % self.server.server.address[1])],
b'',
)))
client.wfile.flush()
@@ -231,7 +231,7 @@ class TestSimple(_Http2Test):
client.wfile,
h2_conn,
headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -272,75 +272,6 @@ class TestSimple(_Http2Test):
@requires_alpn
-class TestForbiddenHeaders(_Http2Test):
-
- @classmethod
- def handle_server_event(cls, event, h2_conn, rfile, wfile):
- if isinstance(event, h2.events.ConnectionTerminated):
- return False
- elif isinstance(event, h2.events.StreamEnded):
- import warnings
- with warnings.catch_warnings():
- # Ignore UnicodeWarning:
- # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison
- # failed to convert both arguments to Unicode - interpreting
- # them as being unequal.
- # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20:
-
- warnings.simplefilter("ignore")
-
- h2_conn.config.validate_outbound_headers = False
- h2_conn.send_headers(event.stream_id, [
- (':status', '200'),
- ('keep-alive', 'foobar'),
- ])
- h2_conn.send_data(event.stream_id, b'response body')
- h2_conn.end_stream(event.stream_id)
- wfile.write(h2_conn.data_to_send())
- wfile.flush()
- return True
-
- def test_forbidden_headers(self):
- client, h2_conn = self._setup_connection()
-
- self._send_request(
- client.wfile,
- h2_conn,
- headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
- (':method', 'GET'),
- (':scheme', 'https'),
- (':path', '/'),
- ])
-
- done = False
- while not done:
- try:
- raw = b''.join(http2.read_raw_frame(client.rfile))
- events = h2_conn.receive_data(raw)
- except exceptions.HttpException:
- print(traceback.format_exc())
- assert False
-
- client.wfile.write(h2_conn.data_to_send())
- client.wfile.flush()
-
- for event in events:
- if isinstance(event, h2.events.ResponseReceived):
- assert 'keep-alive' not in event.headers
- elif isinstance(event, h2.events.StreamEnded):
- done = True
-
- h2_conn.close_connection()
- client.wfile.write(h2_conn.data_to_send())
- client.wfile.flush()
-
- assert len(self.master.state.flows) == 1
- assert self.master.state.flows[0].response.status_code == 200
- assert self.master.state.flows[0].response.headers['keep-alive'] == 'foobar'
-
-
-@requires_alpn
class TestRequestWithPriority(_Http2Test):
@classmethod
@@ -384,7 +315,7 @@ class TestRequestWithPriority(_Http2Test):
client.wfile,
h2_conn,
headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -469,7 +400,7 @@ class TestPriority(_Http2Test):
client.wfile,
h2_conn,
headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -527,7 +458,7 @@ class TestStreamResetFromServer(_Http2Test):
client.wfile,
h2_conn,
headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -576,7 +507,7 @@ class TestBodySizeLimit(_Http2Test):
client.wfile,
h2_conn,
headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -672,7 +603,7 @@ class TestPushPromise(_Http2Test):
client, h2_conn = self._setup_connection()
self._send_request(client.wfile, h2_conn, stream_id=1, headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -728,7 +659,7 @@ class TestPushPromise(_Http2Test):
client, h2_conn = self._setup_connection()
self._send_request(client.wfile, h2_conn, stream_id=1, headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -791,7 +722,7 @@ class TestConnectionLost(_Http2Test):
client, h2_conn = self._setup_connection()
self._send_request(client.wfile, h2_conn, stream_id=1, headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -848,7 +779,7 @@ class TestMaxConcurrentStreams(_Http2Test):
# this will exceed MAX_CONCURRENT_STREAMS on the server connection
# and cause mitmproxy to throttle stream creation to the server
self._send_request(client.wfile, h2_conn, stream_id=stream_id, headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
@@ -894,7 +825,7 @@ class TestConnectionTerminated(_Http2Test):
client, h2_conn = self._setup_connection()
self._send_request(client.wfile, h2_conn, headers=[
- (':authority', "127.0.0.1:{}".format(self.server.server.address.port)),
+ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])),
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py
index 4ea01d34..bac0e527 100644
--- a/test/mitmproxy/proxy/protocol/test_websocket.py
+++ b/test/mitmproxy/proxy/protocol/test_websocket.py
@@ -87,8 +87,8 @@ class _WebSocketTestBase:
"authority",
"CONNECT",
"",
- "localhost",
- self.server.server.address.port,
+ "127.0.0.1",
+ self.server.server.address[1],
"",
"HTTP/1.1",
content=b'')
@@ -105,8 +105,8 @@ class _WebSocketTestBase:
"relative",
"GET",
"http",
- "localhost",
- self.server.server.address.port,
+ "127.0.0.1",
+ self.server.server.address[1],
"/ws",
"HTTP/1.1",
headers=http.Headers(
diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py
index 46beea41..56b09b9a 100644
--- a/test/mitmproxy/proxy/test_server.py
+++ b/test/mitmproxy/proxy/test_server.py
@@ -17,7 +17,6 @@ from mitmproxy.net import socks
from mitmproxy import certs
from mitmproxy import exceptions
from mitmproxy.net.http import http1
-from mitmproxy.net.tcp import Address
from pathod import pathoc
from pathod import pathod
@@ -581,7 +580,7 @@ class TestHttps2Http(tservers.ReverseProxyTest):
def get_options(cls):
opts = super().get_options()
s = parse_server_spec(opts.upstream_server)
- opts.upstream_server = "http://%s" % s.address
+ opts.upstream_server = "http://{}:{}".format(s.address[0], s.address[1])
return opts
def pathoc(self, ssl, sni=None):
@@ -740,7 +739,7 @@ class MasterRedirectRequest(tservers.TestMaster):
# This part should have no impact, but it should also not cause any exceptions.
addr = f.live.server_conn.address
- addr2 = Address(("127.0.0.1", self.redirect_port))
+ addr2 = ("127.0.0.1", self.redirect_port)
f.live.set_server(addr2)
f.live.set_server(addr)
@@ -750,8 +749,8 @@ class MasterRedirectRequest(tservers.TestMaster):
@controller.handler
def response(self, f):
- f.response.content = bytes(f.client_conn.address.port)
- f.response.headers["server-conn-id"] = str(f.server_conn.source_address.port)
+ f.response.content = bytes(f.client_conn.address[1])
+ f.response.headers["server-conn-id"] = str(f.server_conn.source_address[1])
super().response(f)
diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py
index f1eff9ba..9bd3ad25 100644
--- a/test/mitmproxy/test_certs.py
+++ b/test/mitmproxy/test_certs.py
@@ -117,6 +117,12 @@ class TestCertStore:
ret = ca1.get_cert(b"foo.com", [])
assert ret[0].serial == dc[0].serial
+ def test_create_dhparams(self):
+ with tutils.tmpdir() as d:
+ filename = os.path.join(d, "dhparam.pem")
+ certs.CertStore.load_dhparam(filename)
+ assert os.path.exists(filename)
+
class TestDummyCert:
@@ -127,9 +133,10 @@ class TestDummyCert:
ca.default_privatekey,
ca.default_ca,
b"foo.com",
- [b"one.com", b"two.com", b"*.three.com"]
+ [b"one.com", b"two.com", b"*.three.com", b"127.0.0.1"]
)
assert r.cn == b"foo.com"
+ assert r.altnames == [b'one.com', b'two.com', b'*.three.com']
r = certs.dummy_cert(
ca.default_privatekey,
@@ -138,6 +145,7 @@ class TestDummyCert:
[]
)
assert r.cn is None
+ assert r.altnames == []
class TestSSLCert:
@@ -179,3 +187,20 @@ class TestSSLCert:
d = f.read()
s = certs.SSLCert.from_der(d)
assert s.cn
+
+ def test_state(self):
+ with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f:
+ d = f.read()
+ c = certs.SSLCert.from_pem(d)
+
+ c.get_state()
+ c2 = c.copy()
+ a = c.get_state()
+ b = c2.get_state()
+ assert a == b
+ assert c == c2
+ assert c is not c2
+
+ x = certs.SSLCert('')
+ x.set_state(a)
+ assert x == c
diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py
index 777ab4dd..0083f57c 100644
--- a/test/mitmproxy/test_connections.py
+++ b/test/mitmproxy/test_connections.py
@@ -1 +1,210 @@
-# TODO: write tests
+import socket
+import os
+import threading
+import ssl
+import OpenSSL
+import pytest
+from unittest import mock
+
+from mitmproxy import connections
+from mitmproxy import exceptions
+from mitmproxy.net import tcp
+from mitmproxy.net.http import http1
+from mitmproxy.test import tflow
+from mitmproxy.test import tutils
+from .net import tservers
+from pathod import test
+
+
+class TestClientConnection:
+
+ def test_send(self):
+ c = tflow.tclient_conn()
+ c.send(b'foobar')
+ c.send([b'foo', b'bar'])
+ with pytest.raises(TypeError):
+ c.send('string')
+ with pytest.raises(TypeError):
+ c.send(['string', 'not'])
+ assert c.wfile.getvalue() == b'foobarfoobar'
+
+ def test_repr(self):
+ c = tflow.tclient_conn()
+ assert 'address:22' in repr(c)
+ assert 'ALPN' in repr(c)
+ assert 'TLS' not in repr(c)
+
+ c.alpn_proto_negotiated = None
+ c.tls_established = True
+ assert 'ALPN' not in repr(c)
+ assert 'TLS' in repr(c)
+
+ def test_tls_established_property(self):
+ c = tflow.tclient_conn()
+ c.tls_established = True
+ assert c.ssl_established
+ assert c.tls_established
+ c.tls_established = False
+ assert not c.ssl_established
+ assert not c.tls_established
+
+ def test_make_dummy(self):
+ c = connections.ClientConnection.make_dummy(('foobar', 1234))
+ assert c.address == ('foobar', 1234)
+
+ def test_state(self):
+ c = tflow.tclient_conn()
+ assert connections.ClientConnection.from_state(c.get_state()).get_state() == \
+ c.get_state()
+
+ c2 = tflow.tclient_conn()
+ c2.address = (c2.address[0], 4242)
+ assert not c == c2
+
+ c2.timestamp_start = 42
+ c.set_state(c2.get_state())
+ assert c.timestamp_start == 42
+
+ c3 = c.copy()
+ assert c3.get_state() == c.get_state()
+
+
+class TestServerConnection:
+
+ def test_send(self):
+ c = tflow.tserver_conn()
+ c.send(b'foobar')
+ c.send([b'foo', b'bar'])
+ with pytest.raises(TypeError):
+ c.send('string')
+ with pytest.raises(TypeError):
+ c.send(['string', 'not'])
+ assert c.wfile.getvalue() == b'foobarfoobar'
+
+ def test_repr(self):
+ c = tflow.tserver_conn()
+
+ c.sni = 'foobar'
+ c.tls_established = True
+ c.alpn_proto_negotiated = b'h2'
+ assert 'address:22' in repr(c)
+ assert 'ALPN' in repr(c)
+ assert 'TLS: foobar' in repr(c)
+
+ c.sni = None
+ c.tls_established = True
+ c.alpn_proto_negotiated = None
+ assert 'ALPN' not in repr(c)
+ assert 'TLS' in repr(c)
+
+ c.sni = None
+ c.tls_established = False
+ assert 'TLS' not in repr(c)
+
+ def test_tls_established_property(self):
+ c = tflow.tserver_conn()
+ c.tls_established = True
+ assert c.ssl_established
+ assert c.tls_established
+ c.tls_established = False
+ assert not c.ssl_established
+ assert not c.tls_established
+
+ def test_make_dummy(self):
+ c = connections.ServerConnection.make_dummy(('foobar', 1234))
+ assert c.address == ('foobar', 1234)
+
+ def test_simple(self):
+ d = test.Daemon()
+ c = connections.ServerConnection((d.IFACE, d.port))
+ c.connect()
+ f = tflow.tflow()
+ f.server_conn = c
+ f.request.path = "/p/200:da"
+
+ # use this protocol just to assemble - not for actual sending
+ c.wfile.write(http1.assemble_request(f.request))
+ c.wfile.flush()
+
+ assert http1.read_response(c.rfile, f.request, 1000)
+ assert d.last_log()
+
+ c.finish()
+ d.shutdown()
+
+ def test_terminate_error(self):
+ d = test.Daemon()
+ c = connections.ServerConnection((d.IFACE, d.port))
+ c.connect()
+ c.connection = mock.Mock()
+ c.connection.recv = mock.Mock(return_value=False)
+ c.connection.flush = mock.Mock(side_effect=exceptions.TcpDisconnect)
+ c.finish()
+ d.shutdown()
+
+ def test_sni(self):
+ c = connections.ServerConnection(('', 1234))
+ with pytest.raises(ValueError, matches='sni must be str, not '):
+ c.establish_ssl(None, b'foobar')
+
+
+class TestClientConnectionTLS:
+
+ @pytest.mark.parametrize("sni", [
+ None,
+ "example.com"
+ ])
+ def test_tls_with_sni(self, sni):
+ address = ('127.0.0.1', 0)
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.bind(address)
+ sock.listen()
+ address = sock.getsockname()
+
+ def client_run():
+ ctx = ssl.create_default_context()
+ ctx.check_hostname = False
+ ctx.verify_mode = ssl.CERT_NONE
+ s = socket.create_connection(address)
+ s = ctx.wrap_socket(s, server_hostname=sni)
+ s.send(b'foobar')
+ s.shutdown(socket.SHUT_RDWR)
+ threading.Thread(target=client_run).start()
+
+ connection, client_address = sock.accept()
+ c = connections.ClientConnection(connection, client_address, None)
+
+ cert = tutils.test_data.path("mitmproxy/net/data/server.crt")
+ key = OpenSSL.crypto.load_privatekey(
+ OpenSSL.crypto.FILETYPE_PEM,
+ open(tutils.test_data.path("mitmproxy/net/data/server.key"), "rb").read())
+ c.convert_to_ssl(cert, key)
+ assert c.connected()
+ assert c.sni == sni
+ assert c.tls_established
+ assert c.rfile.read(6) == b'foobar'
+ c.finish()
+
+
+class TestServerConnectionTLS(tservers.ServerTestBase):
+ ssl = True
+
+ class handler(tcp.BaseHandler):
+ def handle(self):
+ self.finish()
+
+ @pytest.mark.parametrize("clientcert", [
+ None,
+ tutils.test_data.path("mitmproxy/data/clientcert"),
+ os.path.join(tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem"),
+ ])
+ def test_tls(self, clientcert):
+ c = connections.ServerConnection(("127.0.0.1", self.port))
+ c.connect()
+ c.establish_ssl(clientcert, "foo.com")
+ assert c.connected()
+ assert c.sni == "foo.com"
+ assert c.tls_established
+ c.close()
+ c.finish()
diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py
index fe0f92b3..871d4b9d 100644
--- a/test/mitmproxy/test_eventsequence.py
+++ b/test/mitmproxy/test_eventsequence.py
@@ -32,6 +32,8 @@ def test_websocket_flow(err):
assert len(f.messages) == 1
assert next(i) == ("websocket_message", f)
assert len(f.messages) == 2
+ assert next(i) == ("websocket_message", f)
+ assert len(f.messages) == 3
if err:
assert next(i) == ("websocket_error", f)
assert next(i) == ("websocket_end", f)
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index a78e5f80..0ac3bfd6 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -2,160 +2,18 @@ import io
import pytest
from mitmproxy.test import tflow
-from mitmproxy.net.http import Headers
import mitmproxy.io
from mitmproxy import flowfilter, options
from mitmproxy.contrib import tnetstring
-from mitmproxy.exceptions import FlowReadException, Kill
+from mitmproxy.exceptions import FlowReadException
from mitmproxy import flow
from mitmproxy import http
-from mitmproxy import connections
from mitmproxy.proxy import ProxyConfig
from mitmproxy.proxy.server import DummyServer
from mitmproxy import master
from . import tservers
-class TestHTTPFlow:
-
- def test_copy(self):
- f = tflow.tflow(resp=True)
- f.get_state()
- f2 = f.copy()
- a = f.get_state()
- b = f2.get_state()
- del a["id"]
- del b["id"]
- assert a == b
- assert not f == f2
- assert f is not f2
- assert f.request.get_state() == f2.request.get_state()
- assert f.request is not f2.request
- assert f.request.headers == f2.request.headers
- assert f.request.headers is not f2.request.headers
- assert f.response.get_state() == f2.response.get_state()
- assert f.response is not f2.response
-
- f = tflow.tflow(err=True)
- f2 = f.copy()
- assert f is not f2
- assert f.request is not f2.request
- assert f.request.headers == f2.request.headers
- assert f.request.headers is not f2.request.headers
- assert f.error.get_state() == f2.error.get_state()
- assert f.error is not f2.error
-
- def test_match(self):
- f = tflow.tflow(resp=True)
- assert not flowfilter.match("~b test", f)
- assert flowfilter.match(None, f)
- assert not flowfilter.match("~b test", f)
-
- f = tflow.tflow(err=True)
- assert flowfilter.match("~e", f)
-
- with pytest.raises(ValueError):
- flowfilter.match("~", f)
-
- def test_backup(self):
- f = tflow.tflow()
- f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
- f.request.content = b"foo"
- assert not f.modified()
- f.backup()
- f.request.content = b"bar"
- assert f.modified()
- f.revert()
- assert f.request.content == b"foo"
-
- def test_backup_idempotence(self):
- f = tflow.tflow(resp=True)
- f.backup()
- f.revert()
- f.backup()
- f.revert()
-
- def test_getset_state(self):
- f = tflow.tflow(resp=True)
- state = f.get_state()
- assert f.get_state() == http.HTTPFlow.from_state(
- state).get_state()
-
- f.response = None
- f.error = flow.Error("error")
- state = f.get_state()
- assert f.get_state() == http.HTTPFlow.from_state(
- state).get_state()
-
- f2 = f.copy()
- f2.id = f.id # copy creates a different uuid
- assert f.get_state() == f2.get_state()
- assert not f == f2
- f2.error = flow.Error("e2")
- assert not f == f2
- f.set_state(f2.get_state())
- assert f.get_state() == f2.get_state()
-
- def test_kill(self):
- f = tflow.tflow()
- f.reply.handle()
- f.intercept()
- assert f.killable
- f.kill()
- assert not f.killable
- assert f.reply.value == Kill
-
- def test_resume(self):
- f = tflow.tflow()
- f.reply.handle()
- f.intercept()
- assert f.reply.state == "taken"
- f.resume()
- assert f.reply.state == "committed"
-
- def test_replace_unicode(self):
- f = tflow.tflow(resp=True)
- f.response.content = b"\xc2foo"
- f.replace(b"foo", u"bar")
-
- def test_replace_no_content(self):
- f = tflow.tflow()
- f.request.content = None
- assert f.replace("foo", "bar") == 0
-
- def test_replace(self):
- f = tflow.tflow(resp=True)
- f.request.headers["foo"] = "foo"
- f.request.content = b"afoob"
-
- f.response.headers["foo"] = "foo"
- f.response.content = b"afoob"
-
- assert f.replace("foo", "bar") == 6
-
- assert f.request.headers["bar"] == "bar"
- assert f.request.content == b"abarb"
- assert f.response.headers["bar"] == "bar"
- assert f.response.content == b"abarb"
-
- def test_replace_encoded(self):
- f = tflow.tflow(resp=True)
- f.request.content = b"afoob"
- f.request.encode("gzip")
- f.response.content = b"afoob"
- f.response.encode("gzip")
-
- f.replace("foo", "bar")
-
- assert f.request.raw_content != b"abarb"
- f.request.decode()
- assert f.request.raw_content == b"abarb"
-
- assert f.response.raw_content != b"abarb"
- f.response.decode()
- assert f.response.raw_content == b"abarb"
-
-
class TestSerialize:
def _treader(self):
@@ -307,88 +165,6 @@ class TestFlowMaster:
fm.shutdown()
-class TestRequest:
-
- def test_simple(self):
- f = tflow.tflow()
- r = f.request
- u = r.url
- r.url = u
- with pytest.raises(ValueError):
- setattr(r, "url", "")
- assert r.url == u
- r2 = r.copy()
- assert r.get_state() == r2.get_state()
-
- def test_get_url(self):
- r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
-
- assert r.url == "http://address:22/path"
-
- r.scheme = "https"
- assert r.url == "https://address:22/path"
-
- r.host = "host"
- r.port = 42
- assert r.url == "https://host:42/path"
-
- r.host = "address"
- r.port = 22
- assert r.url == "https://address:22/path"
-
- assert r.pretty_url == "https://address:22/path"
- r.headers["Host"] = "foo.com:22"
- assert r.url == "https://address:22/path"
- assert r.pretty_url == "https://foo.com:22/path"
-
- def test_replace(self):
- r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
- r.path = "path/foo"
- r.headers["Foo"] = "fOo"
- r.content = b"afoob"
- assert r.replace("foo(?i)", "boo") == 4
- assert r.path == "path/boo"
- assert b"foo" not in r.content
- assert r.headers["boo"] == "boo"
-
- def test_constrain_encoding(self):
- r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
- r.headers["accept-encoding"] = "gzip, oink"
- r.constrain_encoding()
- assert "oink" not in r.headers["accept-encoding"]
-
- r.headers.set_all("accept-encoding", ["gzip", "oink"])
- r.constrain_encoding()
- assert "oink" not in r.headers["accept-encoding"]
-
- def test_get_content_type(self):
- resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
- resp.headers = Headers(content_type="text/plain")
- assert resp.headers["content-type"] == "text/plain"
-
-
-class TestResponse:
-
- def test_simple(self):
- f = tflow.tflow(resp=True)
- resp = f.response
- resp2 = resp.copy()
- assert resp2.get_state() == resp.get_state()
-
- def test_replace(self):
- r = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
- r.headers["Foo"] = "fOo"
- r.content = b"afoob"
- assert r.replace("foo(?i)", "boo") == 3
- assert b"foo" not in r.content
- assert r.headers["boo"] == "boo"
-
- def test_get_content_type(self):
- resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
- resp.headers = Headers(content_type="text/plain")
- assert resp.headers["content-type"] == "text/plain"
-
-
class TestError:
def test_getset_state(self):
@@ -409,23 +185,4 @@ class TestError:
def test_repr(self):
e = flow.Error("yay")
assert repr(e)
-
-
-class TestClientConnection:
- def test_state(self):
- c = tflow.tclient_conn()
- assert connections.ClientConnection.from_state(c.get_state()).get_state() == \
- c.get_state()
-
- c2 = tflow.tclient_conn()
- c2.address.address = (c2.address.host, 4242)
- assert not c == c2
-
- c2.timestamp_start = 42
- c.set_state(c2.get_state())
- assert c.timestamp_start == 42
-
- c3 = c.copy()
- assert c3.get_state() == c.get_state()
-
- assert str(c)
+ assert str(e)
diff --git a/test/mitmproxy/test_flowfilter.py b/test/mitmproxy/test_flowfilter.py
index bfce265e..46fff477 100644
--- a/test/mitmproxy/test_flowfilter.py
+++ b/test/mitmproxy/test_flowfilter.py
@@ -1,4 +1,5 @@
import io
+import pytest
from unittest.mock import patch
from mitmproxy.test import tflow
@@ -134,6 +135,12 @@ class TestMatchingHTTPFlow:
e = self.err()
assert self.q("~e", e)
+ def test_fmarked(self):
+ q = self.req()
+ assert not self.q("~marked", q)
+ q.marked = True
+ assert self.q("~marked", q)
+
def test_head(self):
q = self.req()
s = self.resp()
@@ -221,6 +228,11 @@ class TestMatchingHTTPFlow:
assert not self.q("~src :99", q)
assert self.q("~src address:22", q)
+ q.client_conn.address = None
+ assert not self.q('~src address:22', q)
+ q.client_conn = None
+ assert not self.q('~src address:22', q)
+
def test_dst(self):
q = self.req()
q.server_conn = tflow.tserver_conn()
@@ -230,6 +242,11 @@ class TestMatchingHTTPFlow:
assert not self.q("~dst :99", q)
assert self.q("~dst address:22", q)
+ q.server_conn.address = None
+ assert not self.q('~dst address:22', q)
+ q.server_conn = None
+ assert not self.q('~dst address:22', q)
+
def test_and(self):
s = self.resp()
assert self.q("~c 200 & ~h head", s)
@@ -269,6 +286,7 @@ class TestMatchingTCPFlow:
f = self.flow()
assert self.q("~tcp", f)
assert not self.q("~http", f)
+ assert not self.q("~websocket", f)
def test_ferr(self):
e = self.err()
@@ -378,6 +396,87 @@ class TestMatchingTCPFlow:
assert not self.q("~u whatever", f)
+class TestMatchingWebSocketFlow:
+
+ def flow(self):
+ return tflow.twebsocketflow()
+
+ def err(self):
+ return tflow.twebsocketflow(err=True)
+
+ def q(self, q, o):
+ return flowfilter.parse(q)(o)
+
+ def test_websocket(self):
+ f = self.flow()
+ assert self.q("~websocket", f)
+ assert not self.q("~tcp", f)
+ assert not self.q("~http", f)
+
+ def test_ferr(self):
+ e = self.err()
+ assert self.q("~e", e)
+
+ def test_body(self):
+ f = self.flow()
+
+ # Messages sent by client or server
+ assert self.q("~b hello", f)
+ assert self.q("~b me", f)
+ assert not self.q("~b nonexistent", f)
+
+ # Messages sent by client
+ assert self.q("~bq hello", f)
+ assert not self.q("~bq me", f)
+ assert not self.q("~bq nonexistent", f)
+
+ # Messages sent by server
+ assert self.q("~bs me", f)
+ assert not self.q("~bs hello", f)
+ assert not self.q("~bs nonexistent", f)
+
+ def test_src(self):
+ f = self.flow()
+ assert self.q("~src address", f)
+ assert not self.q("~src foobar", f)
+ assert self.q("~src :22", f)
+ assert not self.q("~src :99", f)
+ assert self.q("~src address:22", f)
+
+ def test_dst(self):
+ f = self.flow()
+ f.server_conn = tflow.tserver_conn()
+ assert self.q("~dst address", f)
+ assert not self.q("~dst foobar", f)
+ assert self.q("~dst :22", f)
+ assert not self.q("~dst :99", f)
+ assert self.q("~dst address:22", f)
+
+ def test_and(self):
+ f = self.flow()
+ f.server_conn = tflow.tserver_conn()
+ assert self.q("~b hello & ~b me", f)
+ assert not self.q("~src wrongaddress & ~b hello", f)
+ assert self.q("(~src :22 & ~dst :22) & ~b hello", f)
+ assert not self.q("(~src address:22 & ~dst :22) & ~b nonexistent", f)
+ assert not self.q("(~src address:22 & ~dst :99) & ~b hello", f)
+
+ def test_or(self):
+ f = self.flow()
+ f.server_conn = tflow.tserver_conn()
+ assert self.q("~b hello | ~b me", f)
+ assert self.q("~src :22 | ~b me", f)
+ assert not self.q("~src :99 | ~dst :99", f)
+ assert self.q("(~src :22 | ~dst :22) | ~b me", f)
+
+ def test_not(self):
+ f = self.flow()
+ assert not self.q("! ~src :22", f)
+ assert self.q("! ~src :99", f)
+ assert self.q("!~src :99 !~src :99", f)
+ assert not self.q("!~src :99 !~src :22", f)
+
+
class TestMatchingDummyFlow:
def flow(self):
@@ -411,6 +510,8 @@ class TestMatchingDummyFlow:
assert not self.q("~e", f)
assert not self.q("~http", f)
+ assert not self.q("~tcp", f)
+ assert not self.q("~websocket", f)
assert not self.q("~h whatever", f)
assert not self.q("~hq whatever", f)
@@ -440,3 +541,11 @@ def test_pyparsing_bug(extract_tb):
# The text is a string with leading and trailing whitespace stripped; if the source is not available it is None.
extract_tb.return_value = [("", 1, "test", None)]
assert flowfilter.parse("test")
+
+
+def test_match():
+ with pytest.raises(ValueError):
+ flowfilter.match('[foobar', None)
+
+ assert flowfilter.match(None, None)
+ assert not flowfilter.match('foobar', None)
diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py
index 777ab4dd..889eb0a7 100644
--- a/test/mitmproxy/test_http.py
+++ b/test/mitmproxy/test_http.py
@@ -1 +1,256 @@
-# TODO: write tests
+import pytest
+
+from mitmproxy.test import tflow
+from mitmproxy.net.http import Headers
+import mitmproxy.io
+from mitmproxy import flowfilter
+from mitmproxy.exceptions import Kill
+from mitmproxy import flow
+from mitmproxy import http
+
+
+class TestHTTPRequest:
+
+ def test_simple(self):
+ f = tflow.tflow()
+ r = f.request
+ u = r.url
+ r.url = u
+ with pytest.raises(ValueError):
+ setattr(r, "url", "")
+ assert r.url == u
+ r2 = r.copy()
+ assert r.get_state() == r2.get_state()
+ assert hash(r)
+
+ def test_get_url(self):
+ r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
+
+ assert r.url == "http://address:22/path"
+
+ r.scheme = "https"
+ assert r.url == "https://address:22/path"
+
+ r.host = "host"
+ r.port = 42
+ assert r.url == "https://host:42/path"
+
+ r.host = "address"
+ r.port = 22
+ assert r.url == "https://address:22/path"
+
+ assert r.pretty_url == "https://address:22/path"
+ r.headers["Host"] = "foo.com:22"
+ assert r.url == "https://address:22/path"
+ assert r.pretty_url == "https://foo.com:22/path"
+
+ def test_replace(self):
+ r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
+ r.path = "path/foo"
+ r.headers["Foo"] = "fOo"
+ r.content = b"afoob"
+ assert r.replace("foo(?i)", "boo") == 4
+ assert r.path == "path/boo"
+ assert b"foo" not in r.content
+ assert r.headers["boo"] == "boo"
+
+ def test_constrain_encoding(self):
+ r = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
+ r.headers["accept-encoding"] = "gzip, oink"
+ r.constrain_encoding()
+ assert "oink" not in r.headers["accept-encoding"]
+
+ r.headers.set_all("accept-encoding", ["gzip", "oink"])
+ r.constrain_encoding()
+ assert "oink" not in r.headers["accept-encoding"]
+
+ def test_get_content_type(self):
+ resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
+ resp.headers = Headers(content_type="text/plain")
+ assert resp.headers["content-type"] == "text/plain"
+
+
+class TestHTTPResponse:
+
+ def test_simple(self):
+ f = tflow.tflow(resp=True)
+ resp = f.response
+ resp2 = resp.copy()
+ assert resp2.get_state() == resp.get_state()
+
+ def test_replace(self):
+ r = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
+ r.headers["Foo"] = "fOo"
+ r.content = b"afoob"
+ assert r.replace("foo(?i)", "boo") == 3
+ assert b"foo" not in r.content
+ assert r.headers["boo"] == "boo"
+
+ def test_get_content_type(self):
+ resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
+ resp.headers = Headers(content_type="text/plain")
+ assert resp.headers["content-type"] == "text/plain"
+
+
+class TestHTTPFlow:
+
+ def test_copy(self):
+ f = tflow.tflow(resp=True)
+ assert repr(f)
+ f.get_state()
+ f2 = f.copy()
+ a = f.get_state()
+ b = f2.get_state()
+ del a["id"]
+ del b["id"]
+ assert a == b
+ assert not f == f2
+ assert f is not f2
+ assert f.request.get_state() == f2.request.get_state()
+ assert f.request is not f2.request
+ assert f.request.headers == f2.request.headers
+ assert f.request.headers is not f2.request.headers
+ assert f.response.get_state() == f2.response.get_state()
+ assert f.response is not f2.response
+
+ f = tflow.tflow(err=True)
+ f2 = f.copy()
+ assert f is not f2
+ assert f.request is not f2.request
+ assert f.request.headers == f2.request.headers
+ assert f.request.headers is not f2.request.headers
+ assert f.error.get_state() == f2.error.get_state()
+ assert f.error is not f2.error
+
+ def test_match(self):
+ f = tflow.tflow(resp=True)
+ assert not flowfilter.match("~b test", f)
+ assert flowfilter.match(None, f)
+ assert not flowfilter.match("~b test", f)
+
+ f = tflow.tflow(err=True)
+ assert flowfilter.match("~e", f)
+
+ with pytest.raises(ValueError):
+ flowfilter.match("~", f)
+
+ def test_backup(self):
+ f = tflow.tflow()
+ f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
+ f.request.content = b"foo"
+ assert not f.modified()
+ f.backup()
+ f.request.content = b"bar"
+ assert f.modified()
+ f.revert()
+ assert f.request.content == b"foo"
+
+ def test_backup_idempotence(self):
+ f = tflow.tflow(resp=True)
+ f.backup()
+ f.revert()
+ f.backup()
+ f.revert()
+
+ def test_getset_state(self):
+ f = tflow.tflow(resp=True)
+ state = f.get_state()
+ assert f.get_state() == http.HTTPFlow.from_state(
+ state).get_state()
+
+ f.response = None
+ f.error = flow.Error("error")
+ state = f.get_state()
+ assert f.get_state() == http.HTTPFlow.from_state(
+ state).get_state()
+
+ f2 = f.copy()
+ f2.id = f.id # copy creates a different uuid
+ assert f.get_state() == f2.get_state()
+ assert not f == f2
+ f2.error = flow.Error("e2")
+ assert not f == f2
+ f.set_state(f2.get_state())
+ assert f.get_state() == f2.get_state()
+
+ def test_kill(self):
+ f = tflow.tflow()
+ f.reply.handle()
+ f.intercept()
+ assert f.killable
+ f.kill()
+ assert not f.killable
+ assert f.reply.value == Kill
+
+ def test_resume(self):
+ f = tflow.tflow()
+ f.reply.handle()
+ f.intercept()
+ assert f.reply.state == "taken"
+ f.resume()
+ assert f.reply.state == "committed"
+
+ def test_replace_unicode(self):
+ f = tflow.tflow(resp=True)
+ f.response.content = b"\xc2foo"
+ f.replace(b"foo", u"bar")
+
+ def test_replace_no_content(self):
+ f = tflow.tflow()
+ f.request.content = None
+ assert f.replace("foo", "bar") == 0
+
+ def test_replace(self):
+ f = tflow.tflow(resp=True)
+ f.request.headers["foo"] = "foo"
+ f.request.content = b"afoob"
+
+ f.response.headers["foo"] = "foo"
+ f.response.content = b"afoob"
+
+ assert f.replace("foo", "bar") == 6
+
+ assert f.request.headers["bar"] == "bar"
+ assert f.request.content == b"abarb"
+ assert f.response.headers["bar"] == "bar"
+ assert f.response.content == b"abarb"
+
+ def test_replace_encoded(self):
+ f = tflow.tflow(resp=True)
+ f.request.content = b"afoob"
+ f.request.encode("gzip")
+ f.response.content = b"afoob"
+ f.response.encode("gzip")
+
+ f.replace("foo", "bar")
+
+ assert f.request.raw_content != b"abarb"
+ f.request.decode()
+ assert f.request.raw_content == b"abarb"
+
+ assert f.response.raw_content != b"abarb"
+ f.response.decode()
+ assert f.response.raw_content == b"abarb"
+
+
+def test_make_error_response():
+ resp = http.make_error_response(543, 'foobar', Headers())
+ assert resp
+
+
+def test_make_connect_request():
+ req = http.make_connect_request(('invalidhost', 1234))
+ assert req.first_line_format == 'authority'
+ assert req.method == 'CONNECT'
+ assert req.http_version == 'HTTP/1.1'
+
+
+def test_make_connect_response():
+ resp = http.make_connect_response('foobar')
+ assert resp.http_version == 'foobar'
+ assert resp.status_code == 200
+
+
+def test_expect_continue_response():
+ assert http.expect_continue_response.http_version == 'HTTP/1.1'
+ assert http.expect_continue_response.status_code == 100
diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py
index 65691fdf..161b0dcf 100644
--- a/test/mitmproxy/test_optmanager.py
+++ b/test/mitmproxy/test_optmanager.py
@@ -30,6 +30,14 @@ class TD2(TD):
super().__init__(three=three, **kwargs)
+class TM(optmanager.OptManager):
+ def __init__(self, one="one", two=["foo"], three=None):
+ self.one = one
+ self.two = two
+ self.three = three
+ super().__init__()
+
+
def test_defaults():
assert TD2.default("one") == "done"
assert TD2.default("two") == "dtwo"
@@ -203,6 +211,9 @@ def test_serialize():
t = ""
o2.load(t)
+ with pytest.raises(exceptions.OptionsError, matches='No such option: foobar'):
+ o2.load("foobar: '123'")
+
def test_serialize_defaults():
o = options.Options()
@@ -224,13 +235,10 @@ def test_saving():
o.load_paths(dst)
assert o.three == "foo"
-
-class TM(optmanager.OptManager):
- def __init__(self, one="one", two=["foo"], three=None):
- self.one = one
- self.two = two
- self.three = three
- super().__init__()
+ with open(dst, 'a') as f:
+ f.write("foobar: '123'")
+ with pytest.raises(exceptions.OptionsError, matches=''):
+ o.load_paths(dst)
def test_merge():
diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py
index a14c851e..37cec57a 100644
--- a/test/mitmproxy/test_proxy.py
+++ b/test/mitmproxy/test_proxy.py
@@ -4,62 +4,17 @@ from unittest import mock
from OpenSSL import SSL
import pytest
-from mitmproxy.test import tflow
from mitmproxy.tools import cmdline
from mitmproxy import options
from mitmproxy.proxy import ProxyConfig
-from mitmproxy import connections
from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler
from mitmproxy.proxy import config
-from mitmproxy import exceptions
-from pathod import test
-from mitmproxy.net.http import http1
from mitmproxy.test import tutils
from ..conftest import skip_windows
-class TestServerConnection:
-
- def test_simple(self):
- self.d = test.Daemon()
- sc = connections.ServerConnection((self.d.IFACE, self.d.port))
- sc.connect()
- f = tflow.tflow()
- f.server_conn = sc
- f.request.path = "/p/200:da"
-
- # use this protocol just to assemble - not for actual sending
- sc.wfile.write(http1.assemble_request(f.request))
- sc.wfile.flush()
-
- assert http1.read_response(sc.rfile, f.request, 1000)
- assert self.d.last_log()
-
- sc.finish()
- self.d.shutdown()
-
- def test_terminate_error(self):
- self.d = test.Daemon()
- sc = connections.ServerConnection((self.d.IFACE, self.d.port))
- sc.connect()
- sc.connection = mock.Mock()
- sc.connection.recv = mock.Mock(return_value=False)
- sc.connection.flush = mock.Mock(side_effect=exceptions.TcpDisconnect)
- sc.finish()
- self.d.shutdown()
-
- def test_repr(self):
- sc = tflow.tserver_conn()
- assert "address:22" in repr(sc)
- assert "ssl" not in repr(sc)
- sc.ssl_established = True
- assert "ssl" in repr(sc)
- sc.sni = "foo"
- assert "foo" in repr(sc)
-
-
class MockParser(argparse.ArgumentParser):
"""
@@ -160,7 +115,7 @@ class TestProxyServer:
ProxyServer(conf)
def test_err_2(self):
- conf = ProxyConfig(options.Options(listen_host="invalidhost"))
+ conf = ProxyConfig(options.Options(listen_host="256.256.256.256"))
with pytest.raises(Exception, match="Error starting proxy server"):
ProxyServer(conf)
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index 298fddcb..9a289ae5 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -98,13 +98,14 @@ class ProxyThread(threading.Thread):
threading.Thread.__init__(self)
self.tmaster = tmaster
self.name = "ProxyThread (%s:%s)" % (
- tmaster.server.address.host, tmaster.server.address.port
+ tmaster.server.address[0],
+ tmaster.server.address[1],
)
controller.should_exit = False
@property
def port(self):
- return self.tmaster.server.address.port
+ return self.tmaster.server.address[1]
@property
def tlog(self):
diff --git a/web/src/js/components/FlowView/Details.jsx b/web/src/js/components/FlowView/Details.jsx
index 10ec6553..a73abf37 100644
--- a/web/src/js/components/FlowView/Details.jsx
+++ b/web/src/js/components/FlowView/Details.jsx
@@ -26,7 +26,7 @@ export function ConnectionInfo({ conn }) {
<tbody>
<tr key="address">
<td>Address:</td>
- <td>{conn.address.address.join(':')}</td>
+ <td>{conn.address.join(':')}</td>
</tr>
{conn.sni && (
<tr key="sni">
diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg
index 989bfdd3..7122a1a5 100644
--- a/web/src/js/filt/filt.peg
+++ b/web/src/js/filt/filt.peg
@@ -106,7 +106,7 @@ function destination(regex){
function destinationFilter(flow){
return (!!flow.server_conn.address)
&&
- regex.test(flow.server_conn.address.address[0] + ":" + flow.server_conn.address.address[1]);
+ regex.test(flow.server_conn.address[0] + ":" + flow.server_conn.address[1]);
}
destinationFilter.desc = "destination address matches " + regex;
return destinationFilter;
@@ -172,7 +172,7 @@ function source(regex){
function sourceFilter(flow){
return (!!flow.client_conn.address)
&&
- regex.test(flow.client_conn.address.address[0] + ":" + flow.client_conn.address.address[1]);
+ regex.test(flow.client_conn.address[0] + ":" + flow.client_conn.address[1]);
}
sourceFilter.desc = "source address matches " + regex;
return sourceFilter;