Merge remote-tracking branch 'origin/master' into message-body-encoding

23 files changed, 140 insertions, 100 deletions

diff --git a/examples/add_header.py b/examples/add_header.py
index cf1b53cc..3e0b5f1e 100644
--- a/examples/add_header.py
+++ b/examples/add_header.py
@@ -1,2 +1,2 @@
-def response(context, flow):
+def response(flow):
     flow.response.headers["newheader"] = "foo"
diff --git a/examples/change_upstream_proxy.py b/examples/change_upstream_proxy.py
index 34a6eece..49d5379f 100644
--- a/examples/change_upstream_proxy.py
+++ b/examples/change_upstream_proxy.py
@@ -14,7 +14,7 @@ def proxy_address(flow):
         return ("localhost", 8081)
 
 
-def request(context, flow):
+def request(flow):
     if flow.request.method == "CONNECT":
         # If the decision is done by domain, one could also modify the server address here.
         # We do it after CONNECT here to have the request data available as well.
diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py
index 8a57bf74..5a63e2a0 100644
--- a/examples/custom_contentviews.py
+++ b/examples/custom_contentviews.py
@@ -11,7 +11,7 @@ class ViewPigLatin(contentviews.View):
     content_types = ["text/html"]
 
     def __call__(self, data, **metadata):
-        if strutils.isXML(data):
+        if strutils.is_xml(data):
             parser = lxml.etree.HTMLParser(
                 strip_cdata=True,
                 remove_blank_text=True
@@ -62,9 +62,9 @@ class ViewPigLatin(contentviews.View):
 pig_view = ViewPigLatin()
 
 
-def start(context):
-    context.add_contentview(pig_view)
+def start():
+    contentviews.add(pig_view)
 
 
-def done(context):
-    context.remove_contentview(pig_view)
+def done():
+    contentviews.remove(pig_view)
diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py
index 8d715f33..c020047f 100644
--- a/examples/dns_spoofing.py
+++ b/examples/dns_spoofing.py
@@ -28,7 +28,7 @@ import re
 parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
 
 
-def request(context, flow):
+def request(flow):
     if flow.client_conn.ssl_established:
         flow.request.scheme = "https"
         sni = flow.client_conn.connection.get_servername()
diff --git a/examples/dup_and_replay.py b/examples/dup_and_replay.py
index 9ba91d3b..b47bf951 100644
--- a/examples/dup_and_replay.py
+++ b/examples/dup_and_replay.py
@@ -1,4 +1,7 @@
-def request(context, flow):
-    f = context.duplicate_flow(flow)
+from mitmproxy import master
+
+
+def request(flow):
+    f = master.duplicate_flow(flow)
     f.request.path = "/changed"
-    context.replay_request(f)
+    master.replay_request(f, block=True, run_scripthooks=False)
diff --git a/examples/fail_with_500.py b/examples/fail_with_500.py
index aec85b50..9710f74a 100644
--- a/examples/fail_with_500.py
+++ b/examples/fail_with_500.py
@@ -1,3 +1,3 @@
-def response(context, flow):
+def response(flow):
     flow.response.status_code = 500
     flow.response.content = b""
diff --git a/examples/filt.py b/examples/filt.py
index 1a423845..21744edd 100644
--- a/examples/filt.py
+++ b/examples/filt.py
@@ -3,14 +3,16 @@
 import sys
 from mitmproxy import filt
 
+state = {}
 
-def start(context):
+
+def start():
     if len(sys.argv) != 2:
         raise ValueError("Usage: -s 'filt.py FILTER'")
-    context.filter = filt.parse(sys.argv[1])
+    state["filter"] = filt.parse(sys.argv[1])
 
 
-def response(context, flow):
-    if flow.match(context.filter):
+def response(flow):
+    if flow.match(state["filter"]):
         print("Flow matches filter:")
         print(flow)
diff --git a/examples/flowwriter.py b/examples/flowwriter.py
index cb5ccb0d..07c7ca20 100644
--- a/examples/flowwriter.py
+++ b/examples/flowwriter.py
@@ -3,8 +3,10 @@ import sys
 
 from mitmproxy.flow import FlowWriter
 
+state = {}
 
-def start(context):
+
+def start():
     if len(sys.argv) != 2:
         raise ValueError('Usage: -s "flowriter.py filename"')
 
@@ -12,9 +14,9 @@ def start(context):
         f = sys.stdout
     else:
         f = open(sys.argv[1], "wb")
-    context.flow_writer = FlowWriter(f)
+    state["flow_writer"] = FlowWriter(f)
 
 
-def response(context, flow):
+def response(flow):
     if random.choice([True, False]):
-        context.flow_writer.add(flow)
+        state["flow_writer"].add(flow)
diff --git a/examples/har_extractor.py b/examples/har_extractor.py
index 54aa84d3..76059d8e 100644
--- a/examples/har_extractor.py
+++ b/examples/har_extractor.py
@@ -2,6 +2,7 @@
     This inline script utilizes harparser.HAR from
     https://github.com/JustusW/harparser to generate a HAR log object.
 """
+import mitmproxy.ctx
 import six
 import sys
 import pytz
@@ -54,12 +55,24 @@ class _HARLog(HAR.log):
         return self.__page_list__
 
 
-def start(context):
+class Context(object):
+    pass
+
+context = Context()
+
+
+def start():
     """
         On start we create a HARLog instance. You will have to adapt this to
         suit your actual needs of HAR generation. As it will probably be
         necessary to cluster logs by IPs or reset them from time to time.
     """
+    if sys.version_info >= (3, 0):
+        raise RuntimeError(
+            "har_extractor.py does not work on Python 3. "
+            "Please check out https://github.com/mitmproxy/mitmproxy/issues/1320 "
+            "if you want to help making this work again."
+        )
     context.dump_file = None
     if len(sys.argv) > 1:
         context.dump_file = sys.argv[1]
@@ -73,7 +86,7 @@ def start(context):
     context.seen_server = set()
 
 
-def response(context, flow):
+def response(flow):
     """
        Called when a server response has been received. At the time of this
        message both a request and a response are present and completely done.
@@ -195,7 +208,7 @@ def response(context, flow):
     context.HARLog.add(entry)
 
 
-def done(context):
+def done():
     """
         Called once on script shutdown, after any other events.
     """
@@ -206,17 +219,19 @@ def done(context):
     compressed_json_dump = context.HARLog.compress()
 
     if context.dump_file == '-':
-        context.log(pprint.pformat(json.loads(json_dump)))
+        mitmproxy.ctx.log(pprint.pformat(json.loads(json_dump)))
     elif context.dump_file.endswith('.zhar'):
-        file(context.dump_file, "w").write(compressed_json_dump)
+        with open(context.dump_file, "wb") as f:
+            f.write(compressed_json_dump)
     else:
-        file(context.dump_file, "w").write(json_dump)
-    context.log(
+        with open(context.dump_file, "wb") as f:
+            f.write(json_dump)
+    mitmproxy.ctx.log(
         "HAR log finished with %s bytes (%s bytes compressed)" % (
             len(json_dump), len(compressed_json_dump)
         )
     )
-    context.log(
+    mitmproxy.ctx.log(
         "Compression rate is %s%%" % str(
             100. * len(compressed_json_dump) / len(json_dump)
         )
diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py
index 5803b4c1..352c3c24 100644
--- a/examples/iframe_injector.py
+++ b/examples/iframe_injector.py
@@ -3,24 +3,26 @@
 import sys
 from bs4 import BeautifulSoup
 
+iframe_url = None
 
-def start(context):
+
+def start():
     if len(sys.argv) != 2:
         raise ValueError('Usage: -s "iframe_injector.py url"')
-    context.iframe_url = sys.argv[1]
+    global iframe_url
+    iframe_url = sys.argv[1]
 
 
-def response(context, flow):
-    if flow.request.host in context.iframe_url:
+def response(flow):
+    if flow.request.host in iframe_url:
         return
     html = BeautifulSoup(flow.response.content, "lxml")
     if html.body:
         iframe = html.new_tag(
             "iframe",
-            src=context.iframe_url,
+            src=iframe_url,
             frameborder=0,
             height=0,
             width=0)
         html.body.insert(0, iframe)
-        flow.response.content = str(html)
-        context.log("Iframe inserted.")
+        flow.response.content = str(html).encode("utf8")
diff --git a/examples/modify_form.py b/examples/modify_form.py
index 3fe0cf96..b63a1586 100644
--- a/examples/modify_form.py
+++ b/examples/modify_form.py
@@ -1,4 +1,4 @@
-def request(context, flow):
+def request(flow):
     if flow.request.urlencoded_form:
         flow.request.urlencoded_form["mitmproxy"] = "rocks"
     else:
diff --git a/examples/modify_querystring.py b/examples/modify_querystring.py
index b89e5c8d..ee8a89ad 100644
--- a/examples/modify_querystring.py
+++ b/examples/modify_querystring.py
@@ -1,2 +1,2 @@
-def request(context, flow):
+def request(flow):
     flow.request.query["mitmproxy"] = "rocks"
diff --git a/examples/modify_response_body.py b/examples/modify_response_body.py
index 03dfeaa4..b4632248 100644
--- a/examples/modify_response_body.py
+++ b/examples/modify_response_body.py
@@ -3,16 +3,19 @@
 import sys
 
 
-def start(context):
+state = {}
+
+
+def start():
     if len(sys.argv) != 3:
         raise ValueError('Usage: -s "modify_response_body.py old new"')
     # You may want to use Python's argparse for more sophisticated argument
     # parsing.
-    context.old, context.new = sys.argv[1], sys.argv[2]
+    state["old"], state["new"] = sys.argv[1].encode(), sys.argv[2].encode()
 
 
-def response(context, flow):
+def response(flow):
     flow.response.content = flow.response.content.replace(
-        context.old,
-        context.new
+        state["old"],
+        state["new"]
     )
diff --git a/examples/nonblocking.py b/examples/nonblocking.py
index 4609f389..b81478df 100644
--- a/examples/nonblocking.py
+++ b/examples/nonblocking.py
@@ -1,9 +1,10 @@
 import time
+import mitmproxy
 from mitmproxy.script import concurrent
 
 
 @concurrent  # Remove this and see what happens
-def request(context, flow):
-    context.log("handle request: %s%s" % (flow.request.host, flow.request.path))
+def request(flow):
+    mitmproxy.ctx.log("handle request: %s%s" % (flow.request.host, flow.request.path))
     time.sleep(5)
-    context.log("start  request: %s%s" % (flow.request.host, flow.request.path))
+    mitmproxy.ctx.log("start  request: %s%s" % (flow.request.host, flow.request.path))
diff --git a/examples/proxapp.py b/examples/proxapp.py
index 613d3f8b..2935b587 100644
--- a/examples/proxapp.py
+++ b/examples/proxapp.py
@@ -4,6 +4,7 @@ instance, we're using the Flask framework (http://flask.pocoo.org/) to expose
 a single simplest-possible page.
 """
 from flask import Flask
+import mitmproxy
 
 app = Flask("proxapp")
 
@@ -15,10 +16,10 @@ def hello_world():
 
 # Register the app using the magic domain "proxapp" on port 80. Requests to
 # this domain and port combination will now be routed to the WSGI app instance.
-def start(context):
-    context.app_registry.add(app, "proxapp", 80)
+def start():
+    mitmproxy.ctx.master.apps.add(app, "proxapp", 80)
 
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    context.app_registry.add(app, "example.com", 443)
+    mitmproxy.ctx.master.apps.add(app, "example.com", 443)
diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py
index bb1e6952..36594bcd 100644
--- a/examples/redirect_requests.py
+++ b/examples/redirect_requests.py
@@ -5,7 +5,7 @@ from mitmproxy.models import HTTPResponse
 from netlib.http import Headers
 
 
-def request(context, flow):
+def request(flow):
     # pretty_host takes the "Host" header of the request into account,
     # which is useful in transparent mode where we usually only have the IP
     # otherwise.
@@ -15,7 +15,8 @@ def request(context, flow):
         resp = HTTPResponse(
             b"HTTP/1.1", 200, b"OK",
             Headers(Content_Type="text/html"),
-            b"helloworld")
+            b"helloworld"
+        )
         flow.reply.send(resp)
 
     # Method 2: Redirect the request to a different server
diff --git a/examples/sslstrip.py b/examples/sslstrip.py
index 77e91cc9..0be1f020 100644
--- a/examples/sslstrip.py
+++ b/examples/sslstrip.py
@@ -1,23 +1,21 @@
 import re
 from six.moves import urllib
 
+# set of SSL/TLS capable hosts
+secure_hosts = set()
 
-def start(context):
-    # set of SSL/TLS capable hosts
-    context.secure_hosts = set()
 
-
-def request(context, flow):
+def request(flow):
     flow.request.headers.pop('If-Modified-Since', None)
     flow.request.headers.pop('Cache-Control', None)
 
     # proxy connections to SSL-enabled hosts
-    if flow.request.pretty_host in context.secure_hosts:
+    if flow.request.pretty_host in secure_hosts:
         flow.request.scheme = 'https'
         flow.request.port = 443
 
 
-def response(context, flow):
+def response(flow):
     flow.request.headers.pop('Strict-Transport-Security', None)
     flow.request.headers.pop('Public-Key-Pins', None)
 
@@ -29,7 +27,7 @@ def response(context, flow):
         location = flow.response.headers['Location']
         hostname = urllib.parse.urlparse(location).hostname
         if hostname:
-            context.secure_hosts.add(hostname)
+            secure_hosts.add(hostname)
         flow.response.headers['Location'] = location.replace('https://', 'http://', 1)
 
     # strip secure flag from 'Set-Cookie' headers
diff --git a/examples/stream.py b/examples/stream.py
index 3adbe437..8598f329 100644
--- a/examples/stream.py
+++ b/examples/stream.py
@@ -1,4 +1,4 @@
-def responseheaders(context, flow):
+def responseheaders(flow):
     """
     Enables streaming for all responses.
     """
diff --git a/examples/stream_modify.py b/examples/stream_modify.py
index aa395c03..5e5da95b 100644
--- a/examples/stream_modify.py
+++ b/examples/stream_modify.py
@@ -16,5 +16,5 @@ def modify(chunks):
         yield chunk.replace("foo", "bar")
 
 
-def responseheaders(context, flow):
+def responseheaders(flow):
     flow.response.stream = modify
diff --git a/examples/stub.py b/examples/stub.py
index a0f73538..e5b4a39a 100644
--- a/examples/stub.py
+++ b/examples/stub.py
@@ -1,79 +1,87 @@
+import mitmproxy
 """
     This is a script stub, with definitions for all events.
 """
 
 
-def start(context):
+def start():
     """
-        Called once on script startup, before any other events.
+        Called once on script startup before any other events
     """
-    context.log("start")
+    mitmproxy.ctx.log("start")
 
 
-def clientconnect(context, root_layer):
+def configure(options):
+    """
+        Called once on script startup before any other events, and whenever options changes.
+    """
+    mitmproxy.ctx.log("configure")
+
+
+def clientconnect(root_layer):
     """
         Called when a client initiates a connection to the proxy. Note that a
         connection can correspond to multiple HTTP requests
     """
-    context.log("clientconnect")
+    mitmproxy.ctx.log("clientconnect")
 
 
-def request(context, flow):
+def request(flow):
     """
         Called when a client request has been received.
     """
-    context.log("request")
+    mitmproxy.ctx.log("request")
 
 
-def serverconnect(context, server_conn):
+def serverconnect(server_conn):
     """
         Called when the proxy initiates a connection to the target server. Note that a
         connection can correspond to multiple HTTP requests
     """
-    context.log("serverconnect")
+    mitmproxy.ctx.log("serverconnect")
 
 
-def responseheaders(context, flow):
+def responseheaders(flow):
     """
         Called when the response headers for a server response have been received,
         but the response body has not been processed yet. Can be used to tell mitmproxy
         to stream the response.
     """
-    context.log("responseheaders")
+    mitmproxy.ctx.log("responseheaders")
 
 
-def response(context, flow):
+def response(flow):
     """
        Called when a server response has been received.
     """
-    context.log("response")
+    mitmproxy.ctx.log("response")
 
 
-def error(context, flow):
+def error(flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
         interrupted connections. This is distinct from a valid server HTTP error
         response, which is simply a response with an HTTP error code.
     """
-    context.log("error")
+    mitmproxy.ctx.log("error")
 
 
-def serverdisconnect(context, server_conn):
+def serverdisconnect(server_conn):
     """
         Called when the proxy closes the connection to the target server.
     """
-    context.log("serverdisconnect")
+    mitmproxy.ctx.log("serverdisconnect")
 
 
-def clientdisconnect(context, root_layer):
+def clientdisconnect(root_layer):
     """
         Called when a client disconnects from the proxy.
     """
-    context.log("clientdisconnect")
+    mitmproxy.ctx.log("clientdisconnect")
 
 
-def done(context):
+def done():
     """
         Called once on script shutdown, after any other events.
     """
-    context.log("done")
+    mitmproxy.ctx.log("done")
diff --git a/examples/tcp_message.py b/examples/tcp_message.py
index 78500c19..b431c23f 100644
--- a/examples/tcp_message.py
+++ b/examples/tcp_message.py
@@ -11,15 +11,17 @@ mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py
 from netlib import strutils
 
 
-def tcp_message(ctx, tcp_msg):
+def tcp_message(tcp_msg):
     modified_msg = tcp_msg.message.replace("foo", "bar")
 
     is_modified = False if modified_msg == tcp_msg.message else True
     tcp_msg.message = modified_msg
 
-    print("[tcp_message{}] from {} {} to {} {}:\r\n{}".format(
-        " (modified)" if is_modified else "",
-        "client" if tcp_msg.sender == tcp_msg.client_conn else "server",
-        tcp_msg.sender.address,
-        "server" if tcp_msg.receiver == tcp_msg.server_conn else "client",
-        tcp_msg.receiver.address, strutils.clean_bin(tcp_msg.message)))
+    print(
+        "[tcp_message{}] from {} {} to {} {}:\r\n{}".format(
+            " (modified)" if is_modified else "",
+            "client" if tcp_msg.sender == tcp_msg.client_conn else "server",
+            tcp_msg.sender.address,
+            "server" if tcp_msg.receiver == tcp_msg.server_conn else "client",
+            tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message))
+    )
diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py
index 50aab65b..20e8f9be 100644
--- a/examples/tls_passthrough.py
+++ b/examples/tls_passthrough.py
@@ -20,13 +20,14 @@ Example:
 
 Authors: Maximilian Hils, Matthew Tuusberg
 """
-from __future__ import (absolute_import, print_function, division)
+from __future__ import absolute_import, print_function, division
 import collections
 import random
 
 import sys
 from enum import Enum
 
+import mitmproxy
 from mitmproxy.exceptions import TlsProtocolException
 from mitmproxy.protocol import TlsLayer, RawTCPLayer
 
@@ -97,7 +98,6 @@ class TlsFeedback(TlsLayer):
 
     def _establish_tls_with_client(self):
         server_address = self.server_conn.address
-        tls_strategy = self.script_context.tls_strategy
 
         try:
             super(TlsFeedback, self)._establish_tls_with_client()
@@ -110,15 +110,18 @@ class TlsFeedback(TlsLayer):
 
 # inline script hooks below.
 
+tls_strategy = None
 
-def start(context):
+
+def start():
+    global tls_strategy
     if len(sys.argv) == 2:
-        context.tls_strategy = ProbabilisticStrategy(float(sys.argv[1]))
+        tls_strategy = ProbabilisticStrategy(float(sys.argv[1]))
     else:
-        context.tls_strategy = ConservativeStrategy()
+        tls_strategy = ConservativeStrategy()
 
 
-def next_layer(context, next_layer):
+def next_layer(next_layer):
     """
     This hook does the actual magic - if the next layer is planned to be a TLS layer,
     we check if we want to enter pass-through mode instead.
@@ -126,14 +129,13 @@ def next_layer(context, next_layer):
     if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
         server_address = next_layer.server_conn.address
 
-        if context.tls_strategy.should_intercept(server_address):
+        if tls_strategy.should_intercept(server_address):
             # We try to intercept.
             # Monkey-Patch the layer to get feedback from the TLSLayer if interception worked.
             next_layer.__class__ = TlsFeedback
-            next_layer.script_context = context
         else:
             # We don't intercept - reply with a pass-through layer and add a "skipped" entry.
-            context.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info")
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, logging=False)
+            mitmproxy.ctx.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info")
+            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
             next_layer.reply.send(next_layer_replacement)
-            context.tls_strategy.record_skipped(server_address)
+            tls_strategy.record_skipped(server_address)
diff --git a/examples/upsidedownternet.py b/examples/upsidedownternet.py
index 58ed53d7..d5059092 100644
--- a/examples/upsidedownternet.py
+++ b/examples/upsidedownternet.py
@@ -2,7 +2,7 @@ from six.moves import cStringIO as StringIO
 from PIL import Image
 
 
-def response(context, flow):
+def response(flow):
     if flow.response.headers.get("content-type", "").startswith("image"):
         try:
             s = StringIO(flow.response.content)