From 47196e86760122bb21227b8241a3fcd6272d85ac Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 5 Dec 2016 10:09:32 +0100 Subject: tls_parser: allow optional extensions fixes #1816 --- mitmproxy/contrib/tls/__init__.py | 4 - mitmproxy/contrib/tls/_constructs.py | 203 ----------------------------- mitmproxy/contrib/tls_parser.py | 208 ++++++++++++++++++++++++++++++ mitmproxy/proxy/protocol/tls.py | 32 +++-- test/mitmproxy/contrib/test_tls_parser.py | 38 ++++++ test/mitmproxy/protocol/test_tls.py | 26 ++++ 6 files changed, 290 insertions(+), 221 deletions(-) delete mode 100644 mitmproxy/contrib/tls/__init__.py delete mode 100644 mitmproxy/contrib/tls/_constructs.py create mode 100644 mitmproxy/contrib/tls_parser.py create mode 100644 test/mitmproxy/contrib/test_tls_parser.py create mode 100644 test/mitmproxy/protocol/test_tls.py diff --git a/mitmproxy/contrib/tls/__init__.py b/mitmproxy/contrib/tls/__init__.py deleted file mode 100644 index 450986f7..00000000 --- a/mitmproxy/contrib/tls/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - diff --git a/mitmproxy/contrib/tls/_constructs.py b/mitmproxy/contrib/tls/_constructs.py deleted file mode 100644 index 8b3f12af..00000000 --- a/mitmproxy/contrib/tls/_constructs.py +++ /dev/null @@ -1,203 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - - -from construct import ( - Array, - Bytes, - Struct, - VarInt, - Int8ub, - Int16ub, - Int24ub, - Int32ub, - PascalString, - Embedded, - Prefixed, - Range, - GreedyRange, - Switch, - Optional, -) - -ProtocolVersion = "version" / Struct( - "major" / Int8ub, - "minor" / Int8ub, -) - -TLSPlaintext = "TLSPlaintext" / Struct( - "type" / Int8ub, - ProtocolVersion, - "length" / Int16ub, # TODO: Reject packets with length > 2 ** 14 - "fragment" / Bytes(lambda ctx: ctx.length), -) - -TLSCompressed = "TLSCompressed" / Struct( - "type" / Int8ub, - ProtocolVersion, - "length" / Int16ub, # TODO: Reject packets with length > 2 ** 14 + 1024 - "fragment" / Bytes(lambda ctx: ctx.length), -) - -TLSCiphertext = "TLSCiphertext" / Struct( - "type" / Int8ub, - ProtocolVersion, - "length" / Int16ub, # TODO: Reject packets with length > 2 ** 14 + 2048 - "fragment" / Bytes(lambda ctx: ctx.length), -) - -Random = "random" / Struct( - "gmt_unix_time" / Int32ub, - "random_bytes" / Bytes(28), -) - -SessionID = "session_id" / Struct( - "length" / Int8ub, - "session_id" / Bytes(lambda ctx: ctx.length), -) - -CipherSuites = "cipher_suites" / Struct( - "length" / Int16ub, # TODO: Reject packets of length 0 - Array(lambda ctx: ctx.length // 2, "cipher_suites" / Int16ub), -) - -CompressionMethods = "compression_methods" / Struct( - "length" / Int8ub, # TODO: Reject packets of length 0 - Array(lambda ctx: ctx.length, "compression_methods" / Int8ub), -) - -ServerName = Struct( - "type" / Int8ub, - "name" / PascalString("length" / Int16ub), -) - -SNIExtension = Prefixed( - Int16ub, - Struct( - Int16ub, - "server_names" / GreedyRange( - "server_name" / Struct( - "name_type" / Int8ub, - "host_name" / PascalString("length" / Int16ub), - ) - ) - ) -) - -ALPNExtension = Prefixed( - Int16ub, - Struct( - Int16ub, - "alpn_protocols" / GreedyRange( - "name" / PascalString(Int8ub), - ), - ) -) - -UnknownExtension = Struct( - "bytes" / PascalString("length" / Int16ub) -) - -Extension = "Extension" / Struct( - "type" / Int16ub, - Embedded( - Switch( - lambda ctx: ctx.type, - { - 0x00: SNIExtension, - 0x10: ALPNExtension, - }, - default=UnknownExtension - ) - ) -) - -extensions = "extensions" / Struct( - Int16ub, - "extensions" / GreedyRange(Extension) -) - -ClientHello = "ClientHello" / Struct( - ProtocolVersion, - Random, - SessionID, - CipherSuites, - CompressionMethods, - extensions, -) - -ServerHello = "ServerHello" / Struct( - ProtocolVersion, - Random, - SessionID, - "cipher_suite" / Bytes(2), - "compression_method" / Int8ub, - extensions, -) - -ClientCertificateType = "certificate_types" / Struct( - "length" / Int8ub, # TODO: Reject packets of length 0 - Array(lambda ctx: ctx.length, "certificate_types" / Int8ub), -) - -SignatureAndHashAlgorithm = "algorithms" / Struct( - "hash" / Int8ub, - "signature" / Int8ub, -) - -SupportedSignatureAlgorithms = "supported_signature_algorithms" / Struct( - "supported_signature_algorithms_length" / Int16ub, - # TODO: Reject packets of length 0 - Array( - lambda ctx: ctx.supported_signature_algorithms_length / 2, - SignatureAndHashAlgorithm, - ), -) - -DistinguishedName = "certificate_authorities" / Struct( - "length" / Int16ub, - "certificate_authorities" / Bytes(lambda ctx: ctx.length), -) - -CertificateRequest = "CertificateRequest" / Struct( - ClientCertificateType, - SupportedSignatureAlgorithms, - DistinguishedName, -) - -ServerDHParams = "ServerDHParams" / Struct( - "dh_p_length" / Int16ub, - "dh_p" / Bytes(lambda ctx: ctx.dh_p_length), - "dh_g_length" / Int16ub, - "dh_g" / Bytes(lambda ctx: ctx.dh_g_length), - "dh_Ys_length" / Int16ub, - "dh_Ys" / Bytes(lambda ctx: ctx.dh_Ys_length), -) - -PreMasterSecret = "pre_master_secret" / Struct( - ProtocolVersion, - "random_bytes" / Bytes(46), -) - -ASN1Cert = "ASN1Cert" / Struct( - "length" / Int32ub, # TODO: Reject packets with length not in 1..2^24-1 - "asn1_cert" / Bytes(lambda ctx: ctx.length), -) - -Certificate = "Certificate" / Struct( - # TODO: Reject packets with length > 2 ** 24 - 1 - "certificates_length" / Int32ub, - "certificates_bytes" / Bytes(lambda ctx: ctx.certificates_length), -) - -Handshake = "Handshake" / Struct( - "msg_type" / Int8ub, - "length" / Int24ub, - "body" / Bytes(lambda ctx: ctx.length), -) - -Alert = "Alert" / Struct( - "level" / Int8ub, - "description" / Int8ub, -) diff --git a/mitmproxy/contrib/tls_parser.py b/mitmproxy/contrib/tls_parser.py new file mode 100644 index 00000000..61fb3e3e --- /dev/null +++ b/mitmproxy/contrib/tls_parser.py @@ -0,0 +1,208 @@ +# This file originally comes from https://github.com/pyca/tls/blob/master/tls/_constructs.py. +# Modified by the mitmproxy team. + +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + + +from construct import ( + Array, + Bytes, + Struct, + VarInt, + Int8ub, + Int16ub, + Int24ub, + Int32ub, + PascalString, + Embedded, + Prefixed, + Range, + GreedyRange, + Switch, + Optional, +) + +ProtocolVersion = "version" / Struct( + "major" / Int8ub, + "minor" / Int8ub, +) + +TLSPlaintext = "TLSPlaintext" / Struct( + "type" / Int8ub, + ProtocolVersion, + "length" / Int16ub, # TODO: Reject packets with length > 2 ** 14 + "fragment" / Bytes(lambda ctx: ctx.length), +) + +TLSCompressed = "TLSCompressed" / Struct( + "type" / Int8ub, + ProtocolVersion, + "length" / Int16ub, # TODO: Reject packets with length > 2 ** 14 + 1024 + "fragment" / Bytes(lambda ctx: ctx.length), +) + +TLSCiphertext = "TLSCiphertext" / Struct( + "type" / Int8ub, + ProtocolVersion, + "length" / Int16ub, # TODO: Reject packets with length > 2 ** 14 + 2048 + "fragment" / Bytes(lambda ctx: ctx.length), +) + +Random = "random" / Struct( + "gmt_unix_time" / Int32ub, + "random_bytes" / Bytes(28), +) + +SessionID = "session_id" / Struct( + "length" / Int8ub, + "session_id" / Bytes(lambda ctx: ctx.length), +) + +CipherSuites = "cipher_suites" / Struct( + "length" / Int16ub, # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length // 2, "cipher_suites" / Int16ub), +) + +CompressionMethods = "compression_methods" / Struct( + "length" / Int8ub, # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, "compression_methods" / Int8ub), +) + +ServerName = Struct( + "type" / Int8ub, + "name" / PascalString("length" / Int16ub), +) + +SNIExtension = Prefixed( + Int16ub, + Struct( + Int16ub, + "server_names" / GreedyRange( + "server_name" / Struct( + "name_type" / Int8ub, + "host_name" / PascalString("length" / Int16ub), + ) + ) + ) +) + +ALPNExtension = Prefixed( + Int16ub, + Struct( + Int16ub, + "alpn_protocols" / GreedyRange( + "name" / PascalString(Int8ub), + ), + ) +) + +UnknownExtension = Struct( + "bytes" / PascalString("length" / Int16ub) +) + +Extension = "Extension" / Struct( + "type" / Int16ub, + Embedded( + Switch( + lambda ctx: ctx.type, + { + 0x00: SNIExtension, + 0x10: ALPNExtension, + }, + default=UnknownExtension + ) + ) +) + +extensions = "extensions" / Optional( + Struct( + Int16ub, + "extensions" / GreedyRange(Extension) + ) +) + +ClientHello = "ClientHello" / Struct( + ProtocolVersion, + Random, + SessionID, + CipherSuites, + CompressionMethods, + extensions, +) + +ServerHello = "ServerHello" / Struct( + ProtocolVersion, + Random, + SessionID, + "cipher_suite" / Bytes(2), + "compression_method" / Int8ub, + extensions, +) + +ClientCertificateType = "certificate_types" / Struct( + "length" / Int8ub, # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, "certificate_types" / Int8ub), +) + +SignatureAndHashAlgorithm = "algorithms" / Struct( + "hash" / Int8ub, + "signature" / Int8ub, +) + +SupportedSignatureAlgorithms = "supported_signature_algorithms" / Struct( + "supported_signature_algorithms_length" / Int16ub, + # TODO: Reject packets of length 0 + Array( + lambda ctx: ctx.supported_signature_algorithms_length / 2, + SignatureAndHashAlgorithm, + ), +) + +DistinguishedName = "certificate_authorities" / Struct( + "length" / Int16ub, + "certificate_authorities" / Bytes(lambda ctx: ctx.length), +) + +CertificateRequest = "CertificateRequest" / Struct( + ClientCertificateType, + SupportedSignatureAlgorithms, + DistinguishedName, +) + +ServerDHParams = "ServerDHParams" / Struct( + "dh_p_length" / Int16ub, + "dh_p" / Bytes(lambda ctx: ctx.dh_p_length), + "dh_g_length" / Int16ub, + "dh_g" / Bytes(lambda ctx: ctx.dh_g_length), + "dh_Ys_length" / Int16ub, + "dh_Ys" / Bytes(lambda ctx: ctx.dh_Ys_length), +) + +PreMasterSecret = "pre_master_secret" / Struct( + ProtocolVersion, + "random_bytes" / Bytes(46), +) + +ASN1Cert = "ASN1Cert" / Struct( + "length" / Int32ub, # TODO: Reject packets with length not in 1..2^24-1 + "asn1_cert" / Bytes(lambda ctx: ctx.length), +) + +Certificate = "Certificate" / Struct( + # TODO: Reject packets with length > 2 ** 24 - 1 + "certificates_length" / Int32ub, + "certificates_bytes" / Bytes(lambda ctx: ctx.certificates_length), +) + +Handshake = "Handshake" / Struct( + "msg_type" / Int8ub, + "length" / Int24ub, + "body" / Bytes(lambda ctx: ctx.length), +) + +Alert = "Alert" / Struct( + "level" / Int8ub, + "description" / Int8ub, +) diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 58d9e28d..08ce53d0 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -4,7 +4,7 @@ from typing import Union import construct from mitmproxy import exceptions -from mitmproxy.contrib.tls import _constructs +from mitmproxy.contrib import tls_parser from mitmproxy.proxy.protocol import base from mitmproxy.net import check @@ -248,7 +248,7 @@ def get_client_hello(client_conn): class TlsClientHello: def __init__(self, raw_client_hello): - self._client_hello = _constructs.ClientHello.parse(raw_client_hello) + self._client_hello = tls_parser.ClientHello.parse(raw_client_hello) def raw(self): return self._client_hello @@ -259,21 +259,25 @@ class TlsClientHello: @property def sni(self): - for extension in self._client_hello.extensions.extensions: - is_valid_sni_extension = ( - extension.type == 0x00 and - len(extension.server_names) == 1 and - extension.server_names[0].name_type == 0 and - check.is_valid_host(extension.server_names[0].host_name) - ) - if is_valid_sni_extension: - return extension.server_names[0].host_name.decode("idna") + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + is_valid_sni_extension = ( + extension.type == 0x00 and + len(extension.server_names) == 1 and + extension.server_names[0].name_type == 0 and + check.is_valid_host(extension.server_names[0].host_name) + ) + if is_valid_sni_extension: + return extension.server_names[0].host_name.decode("idna") + return None @property def alpn_protocols(self): - for extension in self._client_hello.extensions.extensions: - if extension.type == 0x10: - return list(extension.alpn_protocols) + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + if extension.type == 0x10: + return list(extension.alpn_protocols) + return [] @classmethod def from_client_conn(cls, client_conn): diff --git a/test/mitmproxy/contrib/test_tls_parser.py b/test/mitmproxy/contrib/test_tls_parser.py new file mode 100644 index 00000000..66972b62 --- /dev/null +++ b/test/mitmproxy/contrib/test_tls_parser.py @@ -0,0 +1,38 @@ +from mitmproxy.contrib import tls_parser + + +def test_parse_chrome(): + """ + Test if we properly parse a ClientHello sent by Chrome 54. + """ + data = bytes.fromhex( + "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" + "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" + "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" + "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" + "170018" + ) + c = tls_parser.ClientHello.parse(data) + assert c.version.major == 3 + assert c.version.minor == 3 + + alpn = [a for a in c.extensions.extensions if a.type == 16] + assert len(alpn) == 1 + assert alpn[0].alpn_protocols == [b"h2", b"http/1.1"] + + sni = [a for a in c.extensions.extensions if a.type == 0] + assert len(sni) == 1 + assert sni[0].server_names[0].name_type == 0 + assert sni[0].server_names[0].host_name == b"example.com" + + +def test_parse_no_extensions(): + data = bytes.fromhex( + "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" + "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" + "61006200640100" + ) + c = tls_parser.ClientHello.parse(data) + assert c.version.major == 3 + assert c.version.minor == 1 + assert c.extensions is None diff --git a/test/mitmproxy/protocol/test_tls.py b/test/mitmproxy/protocol/test_tls.py new file mode 100644 index 00000000..e17ee46f --- /dev/null +++ b/test/mitmproxy/protocol/test_tls.py @@ -0,0 +1,26 @@ +from mitmproxy.proxy.protocol.tls import TlsClientHello + + +class TestClientHello: + + def test_no_extensions(self): + data = bytes.fromhex( + "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" + "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" + "61006200640100" + ) + c = TlsClientHello(data) + assert c.sni is None + assert c.alpn_protocols == [] + + def test_extensions(self): + data = bytes.fromhex( + "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" + "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" + "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" + "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" + "170018" + ) + c = TlsClientHello(data) + assert c.sni == 'example.com' + assert c.alpn_protocols == [b'h2', b'http/1.1'] -- cgit v1.2.3