From 102bd075689892b06765fb857c89604fe9cf33e5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 17 Sep 2014 17:30:19 +0200 Subject: implement FlowStore basics --- libmproxy/protocol/http.py | 2 +- libmproxy/proxy/connection.py | 1 + libmproxy/web/__init__.py | 11 +- libmproxy/web/static/css/app.css | 3 + libmproxy/web/static/js/app.js | 227 ++++++++++++++++++++++++++++++--- libmproxy/web/templates/index.html | 1 - web/gulpfile.js | 3 +- web/src/css/app.less | 4 +- web/src/css/flowtable.less | 5 + web/src/index.html | 1 - web/src/js/actions.js | 22 +++- web/src/js/components/eventlog.jsx | 11 +- web/src/js/components/flowtable.jsx | 121 ++++++++++++++++++ web/src/js/components/proxyapp.jsx | 2 +- web/src/js/components/traffictable.jsx | 34 ----- web/src/js/connection.js | 2 + web/src/js/stores/flowstore.js | 74 +++++++++++ 17 files changed, 458 insertions(+), 66 deletions(-) create mode 100644 web/src/css/flowtable.less create mode 100644 web/src/js/components/flowtable.jsx delete mode 100644 web/src/js/components/traffictable.jsx create mode 100644 web/src/js/stores/flowstore.js diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 1912390a..9db50cd7 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -106,7 +106,7 @@ class HTTPMessage(stateobject.StateObject): timestamp_start=float, timestamp_end=float ) - _stateobject_long_attributes = set(["content"]) + _stateobject_long_attributes = {"content"} def get_decoded_content(self): """ diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index ff36d39c..fd034e8b 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -106,6 +106,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ssl_established=bool, sni=str ) + _stateobject_long_attributes = {"cert"} def get_state(self, short=False): d = super(ServerConnection, self).get_state(short) diff --git a/libmproxy/web/__init__.py b/libmproxy/web/__init__.py index 83a7bde4..c7e0d20d 100644 --- a/libmproxy/web/__init__.py +++ b/libmproxy/web/__init__.py @@ -61,6 +61,8 @@ class WebMaster(flow.FlowMaster): self.app = app.Application(self.options.wdebug) flow.FlowMaster.__init__(self, server, WebState()) + self.last_log_id = 0 + def tick(self): flow.FlowMaster.tick(self, self.masterq, timeout=0) @@ -81,27 +83,30 @@ class WebMaster(flow.FlowMaster): self.shutdown() def handle_request(self, f): - app.ClientConnection.broadcast("flow", f.get_state(True)) + app.ClientConnection.broadcast("add_flow", f.get_state(True)) flow.FlowMaster.handle_request(self, f) if f: f.reply() return f def handle_response(self, f): - app.ClientConnection.broadcast("flow", f.get_state(True)) + s = f.get_state(True) + app.ClientConnection.broadcast("update_flow", f.get_state(True)) flow.FlowMaster.handle_response(self, f) if f: f.reply() return f def handle_error(self, f): - app.ClientConnection.broadcast("flow", f.get_state(True)) + app.ClientConnection.broadcast("update_flow", f.get_state(True)) flow.FlowMaster.handle_error(self, f) return f def handle_log(self, l): + self.last_log_id += 1 app.ClientConnection.broadcast( "add_event", { + "id": self.last_log_id, "message": l.msg, "level": l.level } diff --git a/libmproxy/web/static/css/app.css b/libmproxy/web/static/css/app.css index 6dd17c7d..ecc7426c 100644 --- a/libmproxy/web/static/css/app.css +++ b/libmproxy/web/static/css/app.css @@ -67,6 +67,9 @@ header .menu { padding: 10px; border-bottom: solid #a6a6a6 1px; } +.flow-table { + width: 100%; +} .eventlog { flex: 0 0 auto; margin: 0; diff --git a/libmproxy/web/static/js/app.js b/libmproxy/web/static/js/app.js index 5bc8de0d..e967af89 100644 --- a/libmproxy/web/static/js/app.js +++ b/libmproxy/web/static/js/app.js @@ -49,8 +49,15 @@ AppDispatcher.dispatchServerAction = function (action) { }; var ActionTypes = { + //Settings UPDATE_SETTINGS: "update_settings", - ADD_EVENT: "add_event" + + //EventLog + ADD_EVENT: "add_event", + + //Flow + ADD_FLOW: "add_flow", + UPDATE_FLOW: "update_flow", }; var SettingsActions = { @@ -66,6 +73,18 @@ var SettingsActions = { } }; +var EventLogActions = { + add_event: function(message, level){ + AppDispatcher.dispatchViewAction({ + type: ActionTypes.ADD_EVENT, + data: { + message: message, + level: level || "info", + source: "ui" + } + }); + } +}; function EventEmitter() { this.listeners = {}; } @@ -218,6 +237,81 @@ _.extend(_EventLogStore.prototype, EventEmitter.prototype, { var EventLogStore = new _EventLogStore(); AppDispatcher.register(EventLogStore.handle.bind(EventLogStore)); +function FlowView(store, live) { + EventEmitter.call(this); + this._store = store; + this.live = live; + this.flows = []; + + this.add = this.add.bind(this); + this.update = this.update.bind(this); + + if (live) { + this._store.addListener(ActionTypes.ADD_FLOW, this.add); + this._store.addListener(ActionTypes.UPDATE_FLOW, this.update); + } +} + +_.extend(FlowView.prototype, EventEmitter.prototype, { + close: function () { + this._store.removeListener(ActionTypes.ADD_FLOW, this.add); + this._store.removeListener(ActionTypes.UPDATE_FLOW, this.update); + }, + getAll: function () { + return this.flows; + }, + add: function (flow) { + return this.update(flow); + }, + add_bulk: function (flows) { + //Treat all previously received updates as newer than the bulk update. + //If they weren't newer, we're about to receive an update for them very soon. + var updates = this.flows; + this.flows = flows; + updates.forEach(function(flow){ + this.update(flow); + }.bind(this)); + }, + update: function(flow){ + console.debug("FIXME: Use UUID"); + var idx = _.findIndex(this.flows, function(f){ + return flow.request.timestamp_start == f.request.timestamp_start + }); + + if(idx < 0){ + this.flows.push(flow); + } else { + this.flows[idx] = flow; + } + this.emit("change"); + }, +}); + + +function _FlowStore() { + EventEmitter.call(this); +} +_.extend(_FlowStore.prototype, EventEmitter.prototype, { + getView: function (since) { + var view = new FlowView(this, !since); + return view; + }, + handle: function (action) { + switch (action.type) { + case ActionTypes.ADD_FLOW: + case ActionTypes.UPDATE_FLOW: + this.emit(action.type, action.data); + break; + default: + return; + } + } +}); + + +var FlowStore = new _FlowStore(); +AppDispatcher.register(FlowStore.handle.bind(FlowStore)); + function _Connection(url) { this.url = url; } @@ -242,9 +336,11 @@ _Connection.prototype.onmessage = function (message) { AppDispatcher.dispatchServerAction(m); }; _Connection.prototype.onerror = function (error) { + EventLogActions.add_event("WebSocket Connection Error."); console.log("onerror", this, arguments); }; _Connection.prototype.onclose = function (close) { + EventLogActions.add_event("WebSocket Connection closed."); console.log("onclose", this, arguments); }; @@ -342,35 +438,122 @@ var Header = React.createClass({displayName: 'Header', /** @jsx React.DOM */ -var TrafficTable = React.createClass({displayName: 'TrafficTable', +var FlowRow = React.createClass({displayName: 'FlowRow', + render: function(){ + var flow = this.props.flow; + var columns = this.props.columns.map(function(column){ + return column({flow: flow}); + }.bind(this)); + return React.DOM.tr(null, columns); + } +}); + +var FlowTableHead = React.createClass({displayName: 'FlowTableHead', + render: function(){ + var columns = this.props.columns.map(function(column){ + return column.renderTitle(); + }.bind(this)); + return React.DOM.thead(null, columns); + } +}); + +var FlowTableBody = React.createClass({displayName: 'FlowTableBody', + render: function(){ + var rows = this.props.flows.map(function(flow){ + return FlowRow({flow: flow, columns: this.props.columns}) + }.bind(this)); + return React.DOM.tbody(null, rows); + } +}); + +var PathColumn = React.createClass({displayName: 'PathColumn', + statics: { + renderTitle: function(){ + return React.DOM.th({key: "PathColumn"}, "Path"); + } + }, + render: function(){ + var flow = this.props.flow; + return React.DOM.td({key: "PathColumn"}, flow.request.scheme + "://" + flow.request.host + flow.request.path); + } +}); +var MethodColumn = React.createClass({displayName: 'MethodColumn', + statics: { + renderTitle: function(){ + return React.DOM.th({key: "MethodColumn"}, "Method"); + } + }, + render: function(){ + var flow = this.props.flow; + return React.DOM.td({key: "MethodColumn"}, flow.request.method); + } +}); +var StatusColumn = React.createClass({displayName: 'StatusColumn', + statics: { + renderTitle: function(){ + return React.DOM.th({key: "StatusColumn"}, "Status"); + } + }, + render: function(){ + var flow = this.props.flow; + var status; + if(flow.response){ + status = flow.response.code + " " + flow.response.msg; + } else { + status = null; + } + return React.DOM.td({key: "StatusColumn"}, status); + } +}); +var TimeColumn = React.createClass({displayName: 'TimeColumn', + statics: { + renderTitle: function(){ + return React.DOM.th({key: "TimeColumn"}, "Time"); + } + }, + render: function(){ + var flow = this.props.flow; + var time; + if(flow.response){ + time = Math.round(1000 * (flow.response.timestamp_end - flow.request.timestamp_start))+"ms"; + } else { + time = "..."; + } + return React.DOM.td({key: "TimeColumn"}, time); + } +}); + +var all_columns = [PathColumn, MethodColumn, StatusColumn, TimeColumn]; + +var FlowTable = React.createClass({displayName: 'FlowTable', getInitialState: function () { return { - flows: [] + flows: [], + columns: all_columns }; }, componentDidMount: function () { - //this.flowStore = FlowStore.getView(); - //this.flowStore.addListener("change",this.onFlowChange); + this.flowStore = FlowStore.getView(); + this.flowStore.addListener("change",this.onFlowChange); }, componentWillUnmount: function () { - //this.flowStore.removeListener("change",this.onFlowChange); - //this.flowStore.close(); + this.flowStore.removeListener("change",this.onFlowChange); + this.flowStore.close(); }, onFlowChange: function () { this.setState({ - //flows: this.flowStore.getAll() + flows: this.flowStore.getAll() }); }, render: function () { - /*var flows = this.state.flows.map(function(flow){ - return
{flow.request.method} {flow.request.scheme}://{flow.request.host}{flow.request.path}
; - }); */ - //Dummy Text for layout testing - x = "Flow"; - i = 12; - while (i--) x += x; + var flows = this.state.flows.map(function(flow){ + return React.DOM.div(null, flow.request.method, " ", flow.request.scheme, "://", flow.request.host, flow.request.path); + }); return ( - React.DOM.div(null, "Flow") + React.DOM.table({className: "flow-table"}, + FlowTableHead({columns: this.state.columns}), + FlowTableBody({columns: this.state.columns, flows: this.state.flows}) + ) ); } }); @@ -404,12 +587,18 @@ var EventLog = React.createClass({displayName: 'EventLog', }, render: function () { var messages = this.state.log.map(function(row) { - return (React.DOM.div({key: row.id}, row.message)); + var indicator = null; + if(row.source === "ui"){ + indicator = React.DOM.i({className: "fa fa-html5"}); + } + return ( + React.DOM.div({key: row.id}, + indicator, " ", row.message + )); }); return React.DOM.pre({className: "eventlog"}, messages); } }); - /** @jsx React.DOM */ var Footer = React.createClass({displayName: 'Footer', @@ -463,7 +652,7 @@ var ProxyAppMain = React.createClass({displayName: 'ProxyAppMain', var ProxyApp = ( ReactRouter.Routes({location: "hash"}, ReactRouter.Route({name: "app", path: "/", handler: ProxyAppMain}, - ReactRouter.Route({name: "main", handler: TrafficTable}), + ReactRouter.Route({name: "main", handler: FlowTable}), ReactRouter.Route({name: "reports", handler: Reports}), ReactRouter.Redirect({to: "main"}) ) diff --git a/libmproxy/web/templates/index.html b/libmproxy/web/templates/index.html index 4db2ed51..571845b9 100644 --- a/libmproxy/web/templates/index.html +++ b/libmproxy/web/templates/index.html @@ -10,6 +10,5 @@ -
\ No newline at end of file diff --git a/web/gulpfile.js b/web/gulpfile.js index be44081d..f34bc4a8 100644 --- a/web/gulpfile.js +++ b/web/gulpfile.js @@ -38,9 +38,10 @@ var path = { 'js/stores/base.js', 'js/stores/settingstore.js', 'js/stores/eventlogstore.js', + 'js/stores/flowstore.js', 'js/connection.js', 'js/components/header.jsx', - 'js/components/traffictable.jsx', + 'js/components/flowtable.jsx', 'js/components/eventlog.jsx', 'js/components/footer.jsx', 'js/components/proxyapp.jsx', diff --git a/web/src/css/app.less b/web/src/css/app.less index a5ff516d..1eec0687 100644 --- a/web/src/css/app.less +++ b/web/src/css/app.less @@ -9,6 +9,6 @@ html { @import (less) "layout.less"; @import (less) "header.less"; +@import (less) "flowtable.less"; @import (less) "eventlog.less"; -@import (less) "footer.less"; - +@import (less) "footer.less"; \ No newline at end of file diff --git a/web/src/css/flowtable.less b/web/src/css/flowtable.less new file mode 100644 index 00000000..95f235f4 --- /dev/null +++ b/web/src/css/flowtable.less @@ -0,0 +1,5 @@ +.flow-table { + width: 100%; + + +} \ No newline at end of file diff --git a/web/src/index.html b/web/src/index.html index 4db2ed51..571845b9 100644 --- a/web/src/index.html +++ b/web/src/index.html @@ -10,6 +10,5 @@ -
\ No newline at end of file diff --git a/web/src/js/actions.js b/web/src/js/actions.js index e55a1d16..43fbfb14 100644 --- a/web/src/js/actions.js +++ b/web/src/js/actions.js @@ -1,6 +1,13 @@ var ActionTypes = { + //Settings UPDATE_SETTINGS: "update_settings", - ADD_EVENT: "add_event" + + //EventLog + ADD_EVENT: "add_event", + + //Flow + ADD_FLOW: "add_flow", + UPDATE_FLOW: "update_flow", }; var SettingsActions = { @@ -15,3 +22,16 @@ var SettingsActions = { }); } }; + +var EventLogActions = { + add_event: function(message, level){ + AppDispatcher.dispatchViewAction({ + type: ActionTypes.ADD_EVENT, + data: { + message: message, + level: level || "info", + source: "ui" + } + }); + } +}; \ No newline at end of file diff --git a/web/src/js/components/eventlog.jsx b/web/src/js/components/eventlog.jsx index 32fc01ee..df212177 100644 --- a/web/src/js/components/eventlog.jsx +++ b/web/src/js/components/eventlog.jsx @@ -27,8 +27,15 @@ var EventLog = React.createClass({ }, render: function () { var messages = this.state.log.map(function(row) { - return (
{row.message}
); + var indicator = null; + if(row.source === "ui"){ + indicator = ; + } + return ( +
+ { indicator } {row.message} +
); }); return 
{messages}
; } -}); +}); \ No newline at end of file diff --git a/web/src/js/components/flowtable.jsx b/web/src/js/components/flowtable.jsx new file mode 100644 index 00000000..5e9f6718 --- /dev/null +++ b/web/src/js/components/flowtable.jsx @@ -0,0 +1,121 @@ +/** @jsx React.DOM */ + +var FlowRow = React.createClass({ + render: function(){ + var flow = this.props.flow; + var columns = this.props.columns.map(function(column){ + return column({flow: flow}); + }.bind(this)); + return {columns}; + } +}); + +var FlowTableHead = React.createClass({ + render: function(){ + var columns = this.props.columns.map(function(column){ + return column.renderTitle(); + }.bind(this)); + return {columns}; + } +}); + +var FlowTableBody = React.createClass({ + render: function(){ + var rows = this.props.flows.map(function(flow){ + return + }.bind(this)); + return {rows}; + } +}); + +var PathColumn = React.createClass({ + statics: { + renderTitle: function(){ + return Path; + } + }, + render: function(){ + var flow = this.props.flow; + return {flow.request.scheme + "://" + flow.request.host + flow.request.path}; + } +}); +var MethodColumn = React.createClass({ + statics: { + renderTitle: function(){ + return Method; + } + }, + render: function(){ + var flow = this.props.flow; + return {flow.request.method}; + } +}); +var StatusColumn = React.createClass({ + statics: { + renderTitle: function(){ + return Status; + } + }, + render: function(){ + var flow = this.props.flow; + var status; + if(flow.response){ + status = flow.response.code + " " + flow.response.msg; + } else { + status = null; + } + return {status}; + } +}); +var TimeColumn = React.createClass({ + statics: { + renderTitle: function(){ + return Time; + } + }, + render: function(){ + var flow = this.props.flow; + var time; + if(flow.response){ + time = Math.round(1000 * (flow.response.timestamp_end - flow.request.timestamp_start))+"ms"; + } else { + time = "..."; + } + return {time}; + } +}); + +var all_columns = [PathColumn, MethodColumn, StatusColumn, TimeColumn]; + +var FlowTable = React.createClass({ + getInitialState: function () { + return { + flows: [], + columns: all_columns + }; + }, + componentDidMount: function () { + this.flowStore = FlowStore.getView(); + this.flowStore.addListener("change",this.onFlowChange); + }, + componentWillUnmount: function () { + this.flowStore.removeListener("change",this.onFlowChange); + this.flowStore.close(); + }, + onFlowChange: function () { + this.setState({ + flows: this.flowStore.getAll() + }); + }, + render: function () { + var flows = this.state.flows.map(function(flow){ + return
{flow.request.method} {flow.request.scheme}://{flow.request.host}{flow.request.path}
; + }); + return ( + + + +
+ ); + } +}); diff --git a/web/src/js/components/proxyapp.jsx b/web/src/js/components/proxyapp.jsx index 2f1a9861..63998ffe 100644 --- a/web/src/js/components/proxyapp.jsx +++ b/web/src/js/components/proxyapp.jsx @@ -38,7 +38,7 @@ var ProxyAppMain = React.createClass({ var ProxyApp = ( - + diff --git a/web/src/js/components/traffictable.jsx b/web/src/js/components/traffictable.jsx deleted file mode 100644 index 8071b97e..00000000 --- a/web/src/js/components/traffictable.jsx +++ /dev/null @@ -1,34 +0,0 @@ -/** @jsx React.DOM */ - -var TrafficTable = React.createClass({ - getInitialState: function () { - return { - flows: [] - }; - }, - componentDidMount: function () { - //this.flowStore = FlowStore.getView(); - //this.flowStore.addListener("change",this.onFlowChange); - }, - componentWillUnmount: function () { - //this.flowStore.removeListener("change",this.onFlowChange); - //this.flowStore.close(); - }, - onFlowChange: function () { - this.setState({ - //flows: this.flowStore.getAll() - }); - }, - render: function () { - /*var flows = this.state.flows.map(function(flow){ - return
{flow.request.method} {flow.request.scheme}://{flow.request.host}{flow.request.path}
; - }); */ - //Dummy Text for layout testing - x = "Flow"; - i = 12; - while (i--) x += x; - return ( -
Flow
- ); - } -}); diff --git a/web/src/js/connection.js b/web/src/js/connection.js index 5d464e52..640e9742 100644 --- a/web/src/js/connection.js +++ b/web/src/js/connection.js @@ -22,9 +22,11 @@ _Connection.prototype.onmessage = function (message) { AppDispatcher.dispatchServerAction(m); }; _Connection.prototype.onerror = function (error) { + EventLogActions.add_event("WebSocket Connection Error."); console.log("onerror", this, arguments); }; _Connection.prototype.onclose = function (close) { + EventLogActions.add_event("WebSocket Connection closed."); console.log("onclose", this, arguments); }; diff --git a/web/src/js/stores/flowstore.js b/web/src/js/stores/flowstore.js new file mode 100644 index 00000000..006eeb24 --- /dev/null +++ b/web/src/js/stores/flowstore.js @@ -0,0 +1,74 @@ +function FlowView(store, live) { + EventEmitter.call(this); + this._store = store; + this.live = live; + this.flows = []; + + this.add = this.add.bind(this); + this.update = this.update.bind(this); + + if (live) { + this._store.addListener(ActionTypes.ADD_FLOW, this.add); + this._store.addListener(ActionTypes.UPDATE_FLOW, this.update); + } +} + +_.extend(FlowView.prototype, EventEmitter.prototype, { + close: function () { + this._store.removeListener(ActionTypes.ADD_FLOW, this.add); + this._store.removeListener(ActionTypes.UPDATE_FLOW, this.update); + }, + getAll: function () { + return this.flows; + }, + add: function (flow) { + return this.update(flow); + }, + add_bulk: function (flows) { + //Treat all previously received updates as newer than the bulk update. + //If they weren't newer, we're about to receive an update for them very soon. + var updates = this.flows; + this.flows = flows; + updates.forEach(function(flow){ + this.update(flow); + }.bind(this)); + }, + update: function(flow){ + console.debug("FIXME: Use UUID"); + var idx = _.findIndex(this.flows, function(f){ + return flow.request.timestamp_start == f.request.timestamp_start + }); + + if(idx < 0){ + this.flows.push(flow); + } else { + this.flows[idx] = flow; + } + this.emit("change"); + }, +}); + + +function _FlowStore() { + EventEmitter.call(this); +} +_.extend(_FlowStore.prototype, EventEmitter.prototype, { + getView: function (since) { + var view = new FlowView(this, !since); + return view; + }, + handle: function (action) { + switch (action.type) { + case ActionTypes.ADD_FLOW: + case ActionTypes.UPDATE_FLOW: + this.emit(action.type, action.data); + break; + default: + return; + } + } +}); + + +var FlowStore = new _FlowStore(); +AppDispatcher.register(FlowStore.handle.bind(FlowStore)); -- cgit v1.2.3