aboutsummaryrefslogtreecommitdiffstats
path: root/libmproxy/protocol/http.py
blob: f2265c34024707466ab5baab342ea28f62c73f53 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
from __future__ import (absolute_import, print_function, division)

from netlib import tcp
from netlib.http import http1, HttpErrorConnClosed, HttpError
from netlib.http.semantics import CONTENT_MISSING
from netlib import odict
from netlib.tcp import NetLibError, Address
from netlib.http.http1 import HTTP1Protocol
from netlib.http.http2 import HTTP2Protocol
from netlib.http.http2.frame import Frame, PingFrame, GoAwayFrame, PriorityFrame, WindowUpdateFrame

from .. import utils
from ..exceptions import InvalidCredentials, HttpException, ProtocolException
from ..models import (
    HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error
)
from .base import Layer, Kill


class _HttpLayer(Layer):
    supports_streaming = False

    def read_request(self):
        raise NotImplementedError()

    def send_request(self, request):
        raise NotImplementedError()

    def read_response(self, request_method):
        raise NotImplementedError()

    def send_response(self, response):
        raise NotImplementedError()

    def check_close_connection(self, flow):
        raise NotImplementedError()


class _StreamingHttpLayer(_HttpLayer):
    supports_streaming = True

    def read_response_headers(self):
        raise NotImplementedError

    def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
        raise NotImplementedError()
        yield "this is a generator"  # pragma: no cover

    def send_response_headers(self, response):
        raise NotImplementedError

    def send_response_body(self, response, chunks):
        raise NotImplementedError()


class Http1Layer(_StreamingHttpLayer):
    def __init__(self, ctx, mode):
        super(Http1Layer, self).__init__(ctx)
        self.mode = mode
        self.client_protocol = HTTP1Protocol(self.client_conn)
        self.server_protocol = HTTP1Protocol(self.server_conn)

    def read_request(self):
        return HTTPRequest.from_protocol(
            self.client_protocol,
            body_size_limit=self.config.body_size_limit
        )

    def send_request(self, request):
        self.server_conn.send(self.server_protocol.assemble(request))

    def read_response(self, request_method):
        return HTTPResponse.from_protocol(
            self.server_protocol,
            request_method=request_method,
            body_size_limit=self.config.body_size_limit,
            include_body=True
        )

    def send_response(self, response):
        self.client_conn.send(self.client_protocol.assemble(response))

    def read_response_headers(self):
        return HTTPResponse.from_protocol(
            self.server_protocol,
            request_method=None,  # does not matter if we don't read the body.
            body_size_limit=self.config.body_size_limit,
            include_body=False
        )

    def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
        return self.server_protocol.read_http_body_chunked(
            headers,
            self.config.body_size_limit,
            request_method,
            response_code,
            False,
            max_chunk_size
        )

    def send_response_headers(self, response):
        h = self.client_protocol._assemble_response_first_line(response)
        self.client_conn.wfile.write(h + "\r\n")
        h = self.client_protocol._assemble_response_headers(
            response,
            preserve_transfer_encoding=True
        )
        self.client_conn.send(h + "\r\n")

    def send_response_body(self, response, chunks):
        if self.client_protocol.has_chunked_encoding(response.headers):
            chunks = (
                "%d\r\n%s\r\n" % (len(chunk), chunk)
                for chunk in chunks
            )
        for chunk in chunks:
            self.client_conn.send(chunk)

    def check_close_connection(self, flow):
        close_connection = (
            http1.HTTP1Protocol.connection_close(
                flow.request.httpversion,
                flow.request.headers
            ) or http1.HTTP1Protocol.connection_close(
                flow.response.httpversion,
                flow.response.headers
            ) or http1.HTTP1Protocol.expected_http_body_size(
                flow.response.headers,
                False,
                flow.request.method,
                flow.response.code) == -1
        )
        if flow.request.form_in == "authority" and flow.response.code == 200:
            # Workaround for
            # https://github.com/mitmproxy/mitmproxy/issues/313: Some
            # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
            # and no Content-Length header

            return False
        return close_connection

    def connect(self):
        self.ctx.connect()
        self.server_protocol = HTTP1Protocol(self.server_conn)

    def set_server(self, *args, **kwargs):
        self.ctx.set_server(*args, **kwargs)
        self.server_protocol = HTTP1Protocol(self.server_conn)

    def __call__(self):
        layer = HttpLayer(self, self.mode)
        layer()


# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite.
class Http2Layer(_HttpLayer):
    def __init__(self, ctx, mode):
        super(Http2Layer, self).__init__(ctx)
        self.mode = mode
        self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True,
                                             unhandled_frame_cb=self.handle_unexpected_frame_from_client)
        self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
                                             unhandled_frame_cb=self.handle_unexpected_frame_from_server)

    def read_request(self):
        request = HTTPRequest.from_protocol(
            self.client_protocol,
            body_size_limit=self.config.body_size_limit
        )
        self._stream_id = request.stream_id
        return request

    def send_request(self, message):
        # TODO: implement flow control and WINDOW_UPDATE frames
        self.server_conn.send(self.server_protocol.assemble(message))

    def read_response(self, request_method):
        return HTTPResponse.from_protocol(
            self.server_protocol,
            request_method=request_method,
            body_size_limit=self.config.body_size_limit,
            include_body=True,
            stream_id=self._stream_id
        )

    def send_response(self, message):
        # TODO: implement flow control to prevent client buffer filling up
        # maintain a send buffer size, and read WindowUpdateFrames from client to increase the send buffer
        self.client_conn.send(self.client_protocol.assemble(message))

    def check_close_connection(self, flow):
        # TODO: add a timer to disconnect after a 10 second timeout
        return False

    def connect(self):
        self.ctx.connect()
        self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
                                             unhandled_frame_cb=self.handle_unexpected_frame_from_server)
        self.server_protocol.perform_connection_preface()

    def set_server(self, *args, **kwargs):
        self.ctx.set_server(*args, **kwargs)
        self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False,
                                             unhandled_frame_cb=self.handle_unexpected_frame_from_server)
        self.server_protocol.perform_connection_preface()

    def __call__(self):
        self.server_protocol.perform_connection_preface()
        layer = HttpLayer(self, self.mode)
        layer()

        # terminate the connection
        self.client_conn.send(GoAwayFrame().to_bytes())

    def handle_unexpected_frame_from_client(self, frame):
        if isinstance(frame, WindowUpdateFrame):
            # Clients are sending WindowUpdate frames depending on their flow control algorithm.
            # Since we cannot predict these frames, and we do not need to respond to them,
            # simply accept them, and hide them from the log.
            # Ideally we should keep track of our own flow control window and
            # stall transmission if the outgoing flow control buffer is full.
            return
        if isinstance(frame, PriorityFrame):
            # Clients are sending Priority frames depending on their implementation.
            # The RFC does not clearly state when or which priority preferences should be set.
            # Since we cannot predict these frames, and we do not need to respond to them,
            # simply accept them, and hide them from the log.
            # Ideally we should forward them to the server.
            return
        if isinstance(frame, GoAwayFrame):
            # Client wants to terminate the connection,
            # relay it to the server.
            self.server_conn.send(frame.to_bytes())
            return
        if isinstance(frame, PingFrame):
            # respond with pong
            self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes())
            return
        self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info")

    def handle_unexpected_frame_from_server(self, frame):
        if isinstance(frame, WindowUpdateFrame):
            # Servers are sending WindowUpdate frames depending on their flow control algorithm.
            # Since we cannot predict these frames, and we do not need to respond to them,
            # simply accept them, and hide them from the log.
            # Ideally we should keep track of our own flow control window and
            # stall transmission if the outgoing flow control buffer is full.
            return
        if isinstance(frame, GoAwayFrame):
            # Server wants to terminate the connection,
            # relay it to the client.
            self.client_conn.send(frame.to_bytes())
            return
        if isinstance(frame, PingFrame):
            # respond with pong
            self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes())
            return
        self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info")


class ConnectServerConnection(object):
    """
    "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
    """

    def __init__(self, address, ctx):
        self.address = tcp.Address.wrap(address)
        self._ctx = ctx

    @property
    def via(self):
        return self._ctx.server_conn

    def __getattr__(self, item):
        return getattr(self.via, item)

    def __nonzero__(self):
        return bool(self.via)


class UpstreamConnectLayer(Layer):
    def __init__(self, ctx, connect_request):
        super(UpstreamConnectLayer, self).__init__(ctx)
        self.connect_request = connect_request
        self.server_conn = ConnectServerConnection(
            (connect_request.host, connect_request.port),
            self.ctx
        )

    def __call__(self):
        layer = self.ctx.next_layer(self)
        layer()

    def _send_connect_request(self):
        self.send_request(self.connect_request)
        resp = self.read_response("CONNECT")
        if resp.code != 200:
            raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")

    def connect(self):
        if not self.server_conn:
            self.ctx.connect()
            self._send_connect_request()
        else:
            pass  # swallow the message

    def change_upstream_proxy_server(self, address):
        if address != self.server_conn.via.address:
            self.ctx.set_server(address)

    def set_server(self, address, server_tls=None, sni=None):
        if self.ctx.server_conn:
            self.ctx.disconnect()
        address = Address.wrap(address)
        self.connect_request.host = address.host
        self.connect_request.port = address.port
        self.server_conn.address = address

        if server_tls:
            raise ProtocolException(
                "Cannot upgrade to TLS, no TLS layer on the protocol stack."
            )


class HttpLayer(Layer):
    def __init__(self, ctx, mode):
        super(HttpLayer, self).__init__(ctx)
        self.mode = mode
        self.__original_server_conn = None
        "Contains the original destination in transparent mode, which needs to be restored"
        "if an inline script modified the target server for a single http request"

    def __call__(self):
        if self.mode == "transparent":
            self.__original_server_conn = self.server_conn
        while True:
            try:
                flow = HTTPFlow(self.client_conn, self.server_conn, live=self)

                try:
                    request = self.read_request()
                except tcp.NetLibError:
                    # don't throw an error for disconnects that happen
                    # before/between requests.
                    return

                self.log("request", "debug", [repr(request)])

                # Handle Proxy Authentication
                self.authenticate(request)

                # Regular Proxy Mode: Handle CONNECT
                if self.mode == "regular" and request.form_in == "authority":
                    self.handle_regular_mode_connect(request)
                    return

                # Make sure that the incoming request matches our expectations
                self.validate_request(request)

                flow.request = request
                self.process_request_hook(flow)

                if not flow.response:
                    self.establish_server_connection(flow)
                    self.get_response_from_server(flow)

                self.send_response_to_client(flow)

                if self.check_close_connection(flow):
                    return

                # TODO: Implement HTTP Upgrade

                # Upstream Proxy Mode: Handle CONNECT
                if flow.request.form_in == "authority" and flow.response.code == 200:
                    self.handle_upstream_mode_connect(flow.request.copy())
                    return

            except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e:
                if flow.request and not flow.response:
                    flow.error = Error(repr(e))
                    self.channel.ask("error", flow)
                try:
                    self.send_response(make_error_response(
                        getattr(e, "code", 502),
                        repr(e)
                    ))
                except NetLibError:
                    pass
                if isinstance(e, ProtocolException):
                    raise e
                else:
                    raise ProtocolException("Error in HTTP connection: %s" % repr(e), e)
            finally:
                flow.live = False

    def change_upstream_proxy_server(self, address):
        # Make set_upstream_proxy_server always available,
        # even if there's no UpstreamConnectLayer
        if address != self.server_conn.address:
            return self.set_server(address)

    def handle_regular_mode_connect(self, request):
        self.set_server((request.host, request.port))
        self.send_response(make_connect_response(request.httpversion))
        layer = self.ctx.next_layer(self)
        layer()

    def handle_upstream_mode_connect(self, connect_request):
        layer = UpstreamConnectLayer(self, connect_request)
        layer()

    def send_response_to_client(self, flow):
        if not (self.supports_streaming and flow.response.stream):
            # no streaming:
            # we already received the full response from the server and can
            # send it to the client straight away.
            self.send_response(flow.response)
        else:
            # streaming:
            # First send the headers and then transfer the response incrementally
            self.send_response_headers(flow.response)
            chunks = self.read_response_body(
                flow.response.headers,
                flow.request.method,
                flow.response.code,
                max_chunk_size=4096
            )
            if callable(flow.response.stream):
                chunks = flow.response.stream(chunks)
            self.send_response_body(flow.response, chunks)
            flow.response.timestamp_end = utils.timestamp()

    def get_response_from_server(self, flow):
        def get_response():
            self.send_request(flow.request)
            if self.supports_streaming:
                flow.response = self.read_response_headers()
            else:
                flow.response = self.read_response(flow.request.method)

        try:
            get_response()
        except (tcp.NetLibError, HttpErrorConnClosed) as v:
            self.log(
                "server communication error: %s" % repr(v),
                level="debug"
            )
            # In any case, we try to reconnect at least once. This is
            # necessary because it might be possible that we already
            # initiated an upstream connection after clientconnect that
            # has already been expired, e.g consider the following event
            # log:
            # > clientconnect (transparent mode destination known)
            # > serverconnect (required for client tls handshake)
            # > read n% of large request
            # > server detects timeout, disconnects
            # > read (100-n)% of large request
            # > send large request upstream
            self.disconnect()
            self.connect()
            get_response()

        # call the appropriate script hook - this is an opportunity for an
        # inline script to set flow.stream = True
        flow = self.channel.ask("responseheaders", flow)
        if flow == Kill:
            raise Kill()

        if self.supports_streaming:
            if flow.response.stream:
                flow.response.content = CONTENT_MISSING
            else:
                flow.response.content = "".join(self.read_response_body(
                    flow.response.headers,
                    flow.request.method,
                    flow.response.code
                ))
            flow.response.timestamp_end = utils.timestamp()

        # no further manipulation of self.server_conn beyond this point
        # we can safely set it as the final attribute value here.
        flow.server_conn = self.server_conn

        self.log(
            "response",
            "debug",
            [repr(flow.response)]
        )
        response_reply = self.channel.ask("response", flow)
        if response_reply == Kill:
            raise Kill()

    def process_request_hook(self, flow):
        # Determine .scheme, .host and .port attributes for inline scripts.
        # For absolute-form requests, they are directly given in the request.
        # For authority-form requests, we only need to determine the request scheme.
        # For relative-form requests, we need to determine host and port as
        # well.
        if self.mode == "regular":
            pass  # only absolute-form at this point, nothing to do here.
        elif self.mode == "upstream":
            if flow.request.form_in == "authority":
                flow.request.scheme = "http"  # pseudo value
        else:
            flow.request.host = self.__original_server_conn.address.host
            flow.request.port = self.__original_server_conn.address.port
            flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http"

        request_reply = self.channel.ask("request", flow)
        if request_reply == Kill:
            raise Kill()
        if isinstance(request_reply, HTTPResponse):
            flow.response = request_reply
            return

    def establish_server_connection(self, flow):
        address = tcp.Address((flow.request.host, flow.request.port))
        tls = (flow.request.scheme == "https")

        if self.mode == "regular" or self.mode == "transparent":
            # If there's an existing connection that doesn't match our expectations, kill it.
            if address != self.server_conn.address or tls != self.server_conn.ssl_established:
                self.set_server(address, tls, address.host)
            # Establish connection is neccessary.
            if not self.server_conn:
                self.connect()
        else:
            if not self.server_conn:
                self.connect()
            if tls:
                raise HttpException("Cannot change scheme in upstream proxy mode.")
            """
            # This is a very ugly (untested) workaround to solve a very ugly problem.
            if self.server_conn and self.server_conn.tls_established and not ssl:
                self.disconnect()
                self.connect()
            elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
                if self.server_conn.tls_established:
                    self.disconnect()
                    self.connect()

                self.send_request(make_connect_request(address))
                tls_layer = TlsLayer(self, False, True)
                tls_layer._establish_tls_with_server()
            """

    def validate_request(self, request):
        if request.form_in == "absolute" and request.scheme != "http":
            self.send_response(
                make_error_response(400, "Invalid request scheme: %s" % request.scheme))
            raise HttpException("Invalid request scheme: %s" % request.scheme)

        expected_request_forms = {
            "regular": ("absolute",),  # an authority request would already be handled.
            "upstream": ("authority", "absolute"),
            "transparent": ("relative",)
        }

        allowed_request_forms = expected_request_forms[self.mode]
        if request.form_in not in allowed_request_forms:
            err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
                " or ".join(allowed_request_forms), request.form_in
            )
            self.send_response(make_error_response(400, err_message))
            raise HttpException(err_message)

        if self.mode == "regular":
            request.form_out = "relative"

    def authenticate(self, request):
        if self.config.authenticator:
            if self.config.authenticator.authenticate(request.headers):
                self.config.authenticator.clean(request.headers)
            else:
                self.send_response(make_error_response(
                    407,
                    "Proxy Authentication Required",
                    odict.ODictCaseless(
                        [
                            [k, v] for k, v in
                            self.config.authenticator.auth_challenge_headers().items()
                            ])
                ))
                raise InvalidCredentials("Proxy Authentication Required")