introduce revised views, port over changes from multiple_views branch

3 files changed, 252 insertions, 124 deletions

diff --git a/libmproxy/web/__init__.py b/libmproxy/web/__init__.py
index 69971436..f762466a 100644
--- a/libmproxy/web/__init__.py
+++ b/libmproxy/web/__init__.py
@@ -9,9 +9,32 @@ class Stop(Exception):
     pass
 
 
+class WebFlowView(flow.FlowView):
+    def __init__(self, store):
+        super(WebFlowView, self).__init__(store, None)
+
+    def _add(self, f):
+        super(WebFlowView, self)._add(f)
+        app.FlowUpdates.broadcast("add", f.get_state(short=True))
+
+    def _update(self, f):
+        super(WebFlowView, self)._update(f)
+        app.FlowUpdates.broadcast("update", f.get_state(short=True))
+
+    def _remove(self, f):
+        super(WebFlowView, self)._remove(f)
+        app.FlowUpdates.broadcast("remove", f.get_state(short=True))
+
+    def _recalculate(self, flows):
+        super(WebFlowView, self)._recalculate(flows)
+        app.FlowUpdates.broadcast("recalculate", None)
+
+
 class WebState(flow.State):
     def __init__(self):
-        flow.State.__init__(self)
+        super(WebState, self).__init__()
+        self.view._close()
+        self.view = WebFlowView(self.flows)
 
 
 class Options(object):
@@ -58,8 +81,8 @@ class Options(object):
 class WebMaster(flow.FlowMaster):
     def __init__(self, server, options):
         self.options = options
-        self.app = app.Application(self.options.wdebug)
         super(WebMaster, self).__init__(server, WebState())
+        self.app = app.Application(self.state, self.options.wdebug)
 
         self.last_log_id = 0
 
@@ -83,24 +106,17 @@ class WebMaster(flow.FlowMaster):
             self.shutdown()
 
     def handle_request(self, f):
-        app.ClientConnection.broadcast("add_flow", f.get_state(True))
-        flow.FlowMaster.handle_request(self, f)
+        super(WebMaster, self).handle_request(f)
         if f:
             f.reply()
         return f
 
     def handle_response(self, f):
-        app.ClientConnection.broadcast("update_flow", f.get_state(True))
-        flow.FlowMaster.handle_response(self, f)
+        super(WebMaster, self).handle_response(f)
         if f:
             f.reply()
         return f
 
-    def handle_error(self, f):
-        app.ClientConnection.broadcast("update_flow", f.get_state(True))
-        flow.FlowMaster.handle_error(self, f)
-        return f
-
     def handle_log(self, l):
         self.last_log_id += 1
         app.ClientConnection.broadcast(
diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py
index e2765a6d..4fdff783 100644
--- a/libmproxy/web/app.py
+++ b/libmproxy/web/app.py
@@ -3,6 +3,7 @@ import tornado.web
 import tornado.websocket
 import logging
 import json
+from .. import flow
 
 
 class IndexHandler(tornado.web.RequestHandler):
@@ -10,36 +11,53 @@ class IndexHandler(tornado.web.RequestHandler):
         self.render("index.html")
 
 
-class ClientConnection(tornado.websocket.WebSocketHandler):
-    connections = set()
+class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler):
+    connections = None  # raise an error if inherited class doesn't specify its own instance.
 
     def open(self):
-        ClientConnection.connections.add(self)
+        self.connections.add(self)
 
     def on_close(self):
-        ClientConnection.connections.remove(self)
+        self.connections.remove(self)
 
     @classmethod
     def broadcast(cls, type, data):
+        message = json.dumps(
+            {
+                "type": type,
+                "data": data
+            }
+        )
         for conn in cls.connections:
             try:
-                conn.write_message(
-                    json.dumps(
-                        {
-                            "type": type,
-                            "data": data
-                        }
-                    )
-                )
+                conn.write_message(message)
             except:
                 logging.error("Error sending message", exc_info=True)
 
 
+class FlowsHandler(tornado.web.RequestHandler):
+    def get(self):
+        self.write(dict(
+            flows=[f.get_state(short=True) for f in self.application.state.flows]
+        ))
+
+
+class FlowUpdates(WebSocketEventBroadcaster):
+    connections = set()
+
+
+class ClientConnection(WebSocketEventBroadcaster):
+    connections = set()
+
+
 class Application(tornado.web.Application):
-    def __init__(self, debug):
+    def __init__(self, state, debug):
+        self.state = state
         handlers = [
             (r"/", IndexHandler),
             (r"/updates", ClientConnection),
+            (r"/flows", FlowsHandler),
+            (r"/flows/updates", FlowUpdates),
         ]
         settings = dict(
             template_path=os.path.join(os.path.dirname(__file__), "templates"),
diff --git a/libmproxy/web/static/js/app.js b/libmproxy/web/static/js/app.js
index fe317d7f..ddbb14f4 100644
--- a/libmproxy/web/static/js/app.js
+++ b/libmproxy/web/static/js/app.js
@@ -335,132 +335,216 @@ _.extend(_EventLogStore.prototype, EventEmitter.prototype, {
 
 var EventLogStore = new _EventLogStore();
 AppDispatcher.register(EventLogStore.handle.bind(EventLogStore));
-function FlowView(store, live) {
-    EventEmitter.call(this);
-    this._store = store;
-    this.live = live;
-    this.flows = [];
-
-    this.add = this.add.bind(this);
-    this.update = this.update.bind(this);
-
-    if (live) {
-        this._store.addListener(ActionTypes.ADD_FLOW, this.add);
-        this._store.addListener(ActionTypes.UPDATE_FLOW, this.update);
-    }
+function FlowStore(endpoint) {
+    this._views = [];
+    this.reset();
 }
-
-_.extend(FlowView.prototype, EventEmitter.prototype, {
-    close: function () {
-        this._store.removeListener(ActionTypes.ADD_FLOW, this.add);
-        this._store.removeListener(ActionTypes.UPDATE_FLOW, this.update);
+_.extend(FlowStore.prototype, {
+    add: function (flow) {
+        this._pos_map[flow.id] = this._flow_list.length;
+        this._flow_list.push(flow);
+        for (var i = 0; i < this._views.length; i++) {
+            this._views[i].add(flow);
+        }
     },
-    getAll: function () {
-        return this.flows;
+    update: function (flow) {
+        this._flow_list[this._pos_map[flow.id]] = flow;
+        for (var i = 0; i < this._views.length; i++) {
+            this._views[i].update(flow);
+        }
     },
-    add: function (flow) {
-        return this.update(flow);
-    },
-    add_bulk: function (flows) {
-        //Treat all previously received updates as newer than the bulk update.
-        //If they weren't newer, we're about to receive an update for them very soon.
-        var updates = this.flows;
-        this.flows = flows;
-        updates.forEach(function(flow){
-            this._update(flow);
-        }.bind(this));
-        this.emit("change");
+    remove: function (flow_id) {
+        this._flow_list.splice(this._pos_map[flow_id], 1);
+        this._build_map();
+        for (var i = 0; i < this._views.length; i++) {
+            this._views[i].remove(flow_id);
+        }
     },
-    _update: function(flow){
-        var idx = _.findIndex(this.flows, function(f){
-            return flow.id === f.id;
-        });
+    reset: function (flows) {
+        this._flow_list = flows || [];
+        this._build_map();
+        for (var i = 0; i < this._views.length; i++) {
+            this._views[i].recalculate(this._flow_list);
+        }
+    },
+    _build_map: function () {
+        this._pos_map = {};
+        for (var i = 0; i < this._flow_list.length; i++) {
+            var flow = this._flow_list[i];
+            this._pos_map[flow.id] = i;
+        }
+    },
+    open_view: function (filt, sort) {
+        var view = new FlowView(this._flow_list, filt, sort);
+        this._views.push(view);
+        return view;
+    },
+    close_view: function (view) {
+        this._views = _.without(this._views, view);
+    }
+});
+
 
-        if(idx < 0){
-            this.flows.push(flow);
-            //if(this.flows.length > 100){
-            //    this.flows.shift();
-            //}
+function LiveFlowStore(endpoint) {
+    FlowStore.call(this);
+    this.updates_before_init = []; // (empty array is true in js)
+    this.endpoint = endpoint || "/flows";
+    this.conn = new Connection(this.endpoint + "/updates");
+    this.conn.onopen = this._onopen.bind(this);
+    this.conn.onmessage = function (e) {
+        var message = JSON.parse(e.data);
+        this.handle_update(message.type, message.data);
+    }.bind(this);
+}
+_.extend(LiveFlowStore.prototype, FlowStore.prototype, {
+    handle_update: function (type, data) {
+        console.log("LiveFlowStore.handle_update", type, data);
+        if (this.updates_before_init) {
+            console.log("defer update", type, data);
+            this.updates_before_init.push(arguments);
         } else {
-            this.flows[idx] = flow;
+            this[type](data);
         }
     },
-    update: function(flow){
-        this._update(flow);
-        this.emit("change");
+    handle_fetch: function (data) {
+        console.log("Flows fetched.");
+        this.reset(data.flows);
+        var updates = this.updates_before_init;
+        this.updates_before_init = false;
+        for (var i = 0; i < updates.length; i++) {
+            this.handle_update.apply(this, updates[i]);
+        }
+    },
+    _onopen: function () {
+        //Update stream openend, fetch list of flows.
+        console.log("Update Connection opened, fetching flows...");
+        $.getJSON(this.endpoint, this.handle_fetch.bind(this));
     },
 });
 
+function SortByInsertionOrder() {
+    this.i = 0;
+    this.map = {};
+    this.key = this.key.bind(this);
+}
+SortByInsertionOrder.prototype.key = function (flow) {
+    if (!(flow.id in this.map)) {
+        this.i++;
+        this.map[flow.id] = this.i;
+    }
+    return this.map[flow.id];
+};
 
-function _FlowStore() {
+var default_sort = (new SortByInsertionOrder()).key;
+
+function FlowView(flows, filt, sort) {
     EventEmitter.call(this);
+    filt = filt || function (flow) {
+        return true;
+    };
+    sort = sort || default_sort;
+    this.recalculate(flows, filt, sort);
 }
-_.extend(_FlowStore.prototype, EventEmitter.prototype, {
-    getView: function (since) {
-        var view = new FlowView(this, !since);
-
-        $.getJSON("/static/flows.json", function(flows){
-           flows = flows.concat(_.cloneDeep(flows)).concat(_.cloneDeep(flows));
-           var id = 1;
-           flows.forEach(function(flow){
-               flow.id = "uuid-" + id++;
-           });
-           view.add_bulk(flows); 
 
-        });
+_.extend(FlowView.prototype, EventEmitter.prototype, {
+    recalculate: function (flows, filt, sort) {
+        if (filt) {
+            this.filt = filt;
+        }
+        if (sort) {
+            this.sort = sort;
+        }
+        this.flows = flows.filter(this.filt);
+        this.flows.sort(function (a, b) {
+            return this.sort(a) - this.sort(b);
+        }.bind(this));
+        this.emit("recalculate");
+    },
+    add: function (flow) {
+        if (this.filt(flow)) {
+            var idx = _.sortedIndex(this.flows, flow, this.sort);
+            if (idx === this.flows.length) { //happens often, .push is way faster.
+                this.flows.push(flow);
+            } else {
+                this.flows.splice(idx, 0, flow);
+            }
+            this.emit("add", flow, idx);
+        }
+    },
+    update: function (flow) {
+        var idx;
+        var i = this.flows.length;
+        // Search from the back, we usually update the latest flows.
+        while (i--) {
+            if (this.flows[i].id === flow.id) {
+                idx = i;
+                break;
+            }
+        }
 
-        return view;
+        if (idx === -1) { //not contained in list
+            this.add(flow);
+        } else if (!this.filt(flow)) {
+            this.remove(flow.id);
+        } else {
+            if (this.sort(this.flows[idx]) !== this.sort(flow)) { //sortpos has changed
+                this.remove(this.flows[idx]);
+                this.add(flow);
+            } else {
+                this.flows[idx] = flow;
+                this.emit("update", flow, idx);
+            }
+        }
     },
-    handle: function (action) {
-        switch (action.type) {
-            case ActionTypes.ADD_FLOW:
-            case ActionTypes.UPDATE_FLOW:
-                this.emit(action.type, action.data);
+    remove: function (flow_id) {
+        var i = this.flows.length;
+        while (i--) {
+            if (this.flows[i].id === flow_id) {
+                this.flows.splice(i, 1);
+                this.emit("remove", flow_id, i);
                 break;
-            default:
-                return;
+            }
         }
     }
 });
-
-
-var FlowStore = new _FlowStore();
-AppDispatcher.register(FlowStore.handle.bind(FlowStore));
-
-function _Connection(url) {
-    this.url = url;
+function Connection(url) {
+    if(url[0] != "/"){
+        this.url = url;
+    } else {
+        this.url = location.origin.replace("http", "ws") + url;
+    }
+    var ws = new WebSocket(this.url);
+    ws.onopen = function(){
+        this.onopen.apply(this, arguments);
+    }.bind(this);
+    ws.onmessage = function(){
+        this.onmessage.apply(this, arguments);
+    }.bind(this);
+    ws.onerror = function(){
+        this.onerror.apply(this, arguments);
+    }.bind(this);
+    ws.onclose = function(){
+        this.onclose.apply(this, arguments);
+    }.bind(this);
+    this.ws = ws;
 }
-_Connection.prototype.init = function () {
-    this.openWebSocketConnection();
-};
-_Connection.prototype.openWebSocketConnection = function () {
-    this.ws = new WebSocket(this.url.replace("http", "ws"));
-    var ws = this.ws;
-
-    ws.onopen = this.onopen.bind(this);
-    ws.onmessage = this.onmessage.bind(this);
-    ws.onerror = this.onerror.bind(this);
-    ws.onclose = this.onclose.bind(this);
-};
-_Connection.prototype.onopen = function (open) {
+Connection.prototype.onopen = function (open) {
     console.debug("onopen", this, arguments);
 };
-_Connection.prototype.onmessage = function (message) {
-    //AppDispatcher.dispatchServerAction(...);
-    var m = JSON.parse(message.data);
-    AppDispatcher.dispatchServerAction(m);
+Connection.prototype.onmessage = function (message) {
+    console.warn("onmessage (not implemented)", this, message.data);
 };
-_Connection.prototype.onerror = function (error) {
+Connection.prototype.onerror = function (error) {
     EventLogActions.add_event("WebSocket Connection Error.");
     console.debug("onerror", this, arguments);
 };
-_Connection.prototype.onclose = function (close) {
+Connection.prototype.onclose = function (close) {
     EventLogActions.add_event("WebSocket Connection closed.");
     console.debug("onclose", this, arguments);
 };
-
-var Connection = new _Connection(location.origin + "/updates");
-
+Connection.prototype.close = function(){
+    this.ws.close();
+};
 /** @jsx React.DOM */
 
 //React utils. For other utilities, see ../utils.js
@@ -1214,8 +1298,14 @@ var MainView = React.createClass({displayName: 'MainView',
         };
     },
     componentDidMount: function () {
-        this.flowStore = FlowStore.getView();
-        this.flowStore.addListener("change",this.onFlowChange);
+        //FIXME: The store should be global, move out of here.
+        window.flowstore = new LiveFlowStore();
+
+        this.flowStore = window.flowstore.open_view();
+        this.flowStore.addListener("add",this.onFlowChange);
+        this.flowStore.addListener("update",this.onFlowChange);
+        this.flowStore.addListener("remove",this.onFlowChange);
+        this.flowStore.addListener("recalculate",this.onFlowChange);
     },
     componentWillUnmount: function () {
         this.flowStore.removeListener("change",this.onFlowChange);
@@ -1223,7 +1313,7 @@ var MainView = React.createClass({displayName: 'MainView',
     },
     onFlowChange: function () {
         this.setState({
-            flows: this.flowStore.getAll()
+            flows: this.flowStore.flows
         });
     },
     selectDetailTab: function(panel) {
@@ -1518,7 +1608,11 @@ var ProxyApp = (
     )
     );
 $(function () {
-    Connection.init();
-    app = React.renderComponent(ProxyApp, document.body);
+    window.app = React.renderComponent(ProxyApp, document.body);
+    var UpdateConnection = new Connection("/updates");
+    UpdateConnection.onmessage = function (message) {
+        var m = JSON.parse(message.data);
+        AppDispatcher.dispatchServerAction(m);
+    };
 });
 //# sourceMappingURL=app.js.map
\ No newline at end of file