From 7a154e1ae18f12056447ef030b2bb321cfb13111 Mon Sep 17 00:00:00 2001 From: Pedro Worcel Date: Sat, 22 Feb 2014 15:00:10 +1300 Subject: add basic backwards search --- libmproxy/console/flowview.py | 74 ++++++++++++++++-------- test/test_console_contentview.py | 97 ------------------------------- test/test_console_search.py | 121 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+), 122 deletions(-) create mode 100644 test/test_console_search.py diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 6c4a2651..25871b8d 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -273,7 +273,19 @@ class FlowView(common.WWrap): return False - def search(self, search_string): + def search_again(self, backwards=False): + """ + runs the previous search again, forwards or backwards. + """ + last_search_string = self.state.get_flow_setting(self.flow, "last_search_string") + if last_search_string: + message = self.search(last_search_string, backwards) + if message: + self.master.statusbar.message(message) + else: + self.master.statusbar.message("no previous searches have been made") + + def search(self, search_string, backwards=False): """ similar to view_response or view_request, but instead of just displaying the conn, it highlights a word that the user is @@ -301,7 +313,7 @@ class FlowView(common.WWrap): # generate the body, highlight the words and get focus headers, msg, body = self.conn_text_raw(text) try: - body, focus_position = self.search_highlight_text(body, search_string) + body, focus_position = self.search_highlight_text(body, search_string, backwards=backwards) except SearchError: return "Search not supported in this view." @@ -344,7 +356,7 @@ class FlowView(common.WWrap): return (start_line, start_index) - def search_highlight_text(self, text_objects, search_string, looping = False): + def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False): start_line, start_index = self.search_get_start(search_string) i = start_line @@ -352,40 +364,39 @@ class FlowView(common.WWrap): text_objects = copy.deepcopy(text_objects) for text_object in text_objects[start_line:]: if i != start_line: - start_index = 0 + start_index = None try: text, style = text_object.get_text() except AttributeError: raise SearchError() - find_index = text.find(search_string, start_index) + + if backwards == False: + find_index = text.find(search_string, start_index) + else: + if start_index != 0: + start_index -= len(search_string) + + find_index = text.rfind(search_string, 0, start_index) + + # Found text in line, do the highlight highlight. if find_index != -1: - before = text[:find_index] - after = text[find_index+len(search_string):] - new_text = urwid.Text( - [ - before, - (self.highlight_color, search_string), - after, - ] - ) + new_text = self.search_highlight_object(text, find_index, search_string) + text_objects[i] = new_text self.state.add_flow_setting(self.flow, "last_search_index", find_index) self.state.add_flow_setting(self.flow, "last_find_line", i) - text_objects[i] = new_text - found = True - break i += 1 + # handle search WRAP if found: focus_pos = i else : - # loop from the beginning, but not forever. if (start_line == 0 and start_index == 0) or looping: focus_pos = None else: @@ -395,6 +406,23 @@ class FlowView(common.WWrap): return text_objects, focus_pos + def search_highlight_object(self, text_object, find_index, search_string): + """ + just a little abstraction + """ + before = text_object[:find_index] + after = text_object[find_index+len(search_string):] + + new_text = urwid.Text( + [ + before, + (self.highlight_color, search_string), + after, + ] + ) + + return new_text + def view_request(self): self.state.view_flow_mode = common.VIEW_FLOW_REQUEST body = self.conn_text(self.flow.request) @@ -761,13 +789,9 @@ class FlowView(common.WWrap): None, self.search) elif key == "n": - last_search_string = self.state.get_flow_setting(self.flow, "last_search_string") - if last_search_string: - message = self.search(last_search_string) - if message: - self.master.statusbar.message(message) - else: - self.master.statusbar.message("no previous searches have been made") + self.search_again(backwards=False) + elif key == "N": + self.search_again(backwards=True) else: return key diff --git a/test/test_console_contentview.py b/test/test_console_contentview.py index 07ecf1d0..a878ad4e 100644 --- a/test/test_console_contentview.py +++ b/test/test_console_contentview.py @@ -276,100 +276,3 @@ if cv.ViewProtobuf.is_available(): def test_get_by_shortcut(): assert cv.get_by_shortcut("h") -def test_search_highlights(): - # Default text in requests is content. We will search for nt once, and - # expect the first bit to be highlighted. We will do it again and expect the - # second to be. - f = tutils.tflowview() - - f.search("nt") - text_object = tutils.get_body_line(f.last_displayed_body, 0) - assert text_object.get_text() == ('content', [(None, 2), (f.highlight_color, 2)]) - - f.search("nt") - text_object = tutils.get_body_line(f.last_displayed_body, 1) - assert text_object.get_text() == ('content', [(None, 5), (f.highlight_color, 2)]) - -def test_search_returns_useful_messages(): - f = tutils.tflowview() - - # original string is content. this string should not be in there. - response = f.search("oranges and other fruit.") - assert response == "no matches for 'oranges and other fruit.'" - -def test_search_highlights_clears_prev(): - f = tutils.tflowview(request_contents="this is string\nstring is string") - - f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 0) - assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) - - # search again, it should not be highlighted again. - f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 0) - assert text_object.get_text() != ('this is string', [(None, 8), (f.highlight_color, 6)]) - -def test_search_highlights_multi_line(): - f = tutils.tflowview(request_contents="this is string\nstring is string") - - # should highlight the first line. - f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 0) - assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) - - # should highlight second line, first appearance of string. - f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 1) - assert text_object.get_text() == ('string is string', [(None, 0), (f.highlight_color, 6)]) - - # should highlight third line, second appearance of string. - f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 1) - assert text_object.get_text() == ('string is string', [(None, 10), (f.highlight_color, 6)]) - -def test_search_loops(): - f = tutils.tflowview(request_contents="this is string\nstring is string") - - # get to the end. - f.search("string") - f.search("string") - f.search("string") - - # should highlight the first line. - message = f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 0) - assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) - assert message == "search hit BOTTOM, continuing at TOP" - -def test_search_focuses(): - f = tutils.tflowview(request_contents="this is string\nstring is string") - - # should highlight the first line. - f.search("string") - - # should be focusing on the 2nd text line. - f.search("string") - text_object = tutils.get_body_line(f.last_displayed_body, 1) - assert f.last_displayed_body.focus == text_object - -def test_search_does_not_crash_on_bad(): - """ - this used to crash, kept for reference. - """ - - f = tutils.tflowview(request_contents="this is string\nstring is string\n"+("A" * cv.VIEW_CUTOFF)+"AFTERCUTOFF") - f.search("AFTERCUTOFF") - - # pretend F - f.state.add_flow_setting( - f.flow, - (f.state.view_flow_mode, "fullcontents"), - True - ) - f.master.refresh_flow(f.flow) - - # text changed, now this string will exist. can happen when user presses F - # for full text view - f.search("AFTERCUTOFF") - - diff --git a/test/test_console_search.py b/test/test_console_search.py new file mode 100644 index 00000000..7374f399 --- /dev/null +++ b/test/test_console_search.py @@ -0,0 +1,121 @@ + +import sys +import libmproxy.console.contentview as cv +from libmproxy import utils, flow, encoding +import tutils + +def test_search_highlights(): + # Default text in requests is content. We will search for nt once, and + # expect the first bit to be highlighted. We will do it again and expect the + # second to be. + f = tutils.tflowview() + + f.search("nt") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == ('content', [(None, 2), (f.highlight_color, 2)]) + + f.search("nt") + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert text_object.get_text() == ('content', [(None, 5), (f.highlight_color, 2)]) + +def test_search_returns_useful_messages(): + f = tutils.tflowview() + + # original string is content. this string should not be in there. + test_string = "oranges and other fruit." + response = f.search(test_string) + assert response == "no matches for '%s'" % test_string + +def test_search_highlights_clears_prev(): + f = tutils.tflowview(request_contents="this is string\nstring is string") + + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + + # search again, it should not be highlighted again. + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() != ('this is string', [(None, 8), (f.highlight_color, 6)]) + +def test_search_highlights_multi_line(): + f = tutils.tflowview(request_contents="this is string\nstring is string") + + # should highlight the first line. + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + + # should highlight second line, first appearance of string. + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert text_object.get_text() == ('string is string', [(None, 0), (f.highlight_color, 6)]) + + # should highlight third line, second appearance of string. + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 1) + print(text_object.get_text(), ('string is string', [(None, 10), (f.highlight_color, 6)])) + assert text_object.get_text() == ('string is string', [(None, 10), (f.highlight_color, 6)]) + +def test_search_loops(): + f = tutils.tflowview(request_contents="this is string\nstring is string") + + # get to the end. + f.search("string") + f.search("string") + f.search("string") + + # should highlight the first line. + message = f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + assert message == "search hit BOTTOM, continuing at TOP" + +def test_search_focuses(): + f = tutils.tflowview(request_contents="this is string\nstring is string") + + # should highlight the first line. + f.search("string") + + # should be focusing on the 2nd text line. + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert f.last_displayed_body.focus == text_object + +def test_search_does_not_crash_on_bad(): + """ + this used to crash, kept for reference. + """ + + f = tutils.tflowview(request_contents="this is string\nstring is string\n"+("A" * cv.VIEW_CUTOFF)+"AFTERCUTOFF") + f.search("AFTERCUTOFF") + + # pretend F + f.state.add_flow_setting( + f.flow, + (f.state.view_flow_mode, "fullcontents"), + True + ) + f.master.refresh_flow(f.flow) + + # text changed, now this string will exist. can happen when user presses F + # for full text view + f.search("AFTERCUTOFF") + +def test_search_backwards(): + f = tutils.tflowview(request_contents="content, content") + + first_match = ('content, content', [(None, 2), (f.highlight_color, 2)]) + + f.search("nt") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == first_match + + f.search("nt") + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert text_object.get_text() == ('content, content', [(None, 5), (f.highlight_color, 2)]) + + f.search_again(backwards=True) + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == first_match + -- cgit v1.2.3 From 3c02865e8b5839d536bc9982e4c0e6e699fd1943 Mon Sep 17 00:00:00 2001 From: Pedro Worcel Date: Sat, 22 Feb 2014 15:32:35 +1300 Subject: add multi-line support --- libmproxy/console/flowview.py | 13 ++++++++++--- test/test_console_search.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 25871b8d..9cdd2923 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -356,13 +356,22 @@ class FlowView(common.WWrap): return (start_line, start_index) + def search_get_range(self, len_text_objects, start_line, backwards): + if not backwards: + loop_range = range(start_line, len_text_objects) + else: + loop_range = range(start_line, 0, -1) + + return loop_range + def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False): start_line, start_index = self.search_get_start(search_string) i = start_line found = False text_objects = copy.deepcopy(text_objects) - for text_object in text_objects[start_line:]: + for i in self.search_get_range(len(text_objects), start_line, backwards): + text_object = text_objects[i] if i != start_line: start_index = None @@ -391,8 +400,6 @@ class FlowView(common.WWrap): found = True break - i += 1 - # handle search WRAP if found: focus_pos = i diff --git a/test/test_console_search.py b/test/test_console_search.py index 7374f399..69c033b9 100644 --- a/test/test_console_search.py +++ b/test/test_console_search.py @@ -119,3 +119,18 @@ def test_search_backwards(): text_object = tutils.get_body_line(f.last_displayed_body, 0) assert text_object.get_text() == first_match +def test_search_back_multiline(): + f = tutils.tflowview(request_contents="this is string\nstring is string") + + f.search("string") + text_object = tutils.get_body_line(f.last_displayed_body, 0) + first_match = ('this is string', [(None, 8), (f.highlight_color, 6)]) + assert text_object.get_text() == first_match + + f.search_again() + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert text_object.get_text() == ('string is string', [(None, 0), (f.highlight_color, 6)]) + + f.search_again(backwards=True) + text_object = tutils.get_body_line(f.last_displayed_body, 0) + assert text_object.get_text() == first_match -- cgit v1.2.3 From 4284fd3614561fe5e2c53154defb3774f14c589c Mon Sep 17 00:00:00 2001 From: Pedro Worcel Date: Sat, 22 Feb 2014 16:32:22 +1300 Subject: add multi-line support to backwards search --- libmproxy/console/flowview.py | 18 +++++++++++------- test/test_console_search.py | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 9cdd2923..624cf95d 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -330,7 +330,7 @@ class FlowView(common.WWrap): self.last_displayed_body = list_box - if self.search_wrapped_around(last_find_line, last_search_index): + if not backwards and self.search_wrapped_around(last_find_line, last_search_index): return "search hit BOTTOM, continuing at TOP" def search_get_start(self, search_string): @@ -358,9 +358,9 @@ class FlowView(common.WWrap): def search_get_range(self, len_text_objects, start_line, backwards): if not backwards: - loop_range = range(start_line, len_text_objects) + loop_range = xrange(start_line, len_text_objects) else: - loop_range = range(start_line, 0, -1) + loop_range = xrange(start_line, -1, -1) return loop_range @@ -370,10 +370,11 @@ class FlowView(common.WWrap): found = False text_objects = copy.deepcopy(text_objects) - for i in self.search_get_range(len(text_objects), start_line, backwards): + loop_range = self.search_get_range(len(text_objects), start_line, backwards) + for i in loop_range: text_object = text_objects[i] if i != start_line: - start_index = None + start_index = 0 try: text, style = text_object.get_text() @@ -385,16 +386,18 @@ class FlowView(common.WWrap): else: if start_index != 0: start_index -= len(search_string) + else: + start_index = None find_index = text.rfind(search_string, 0, start_index) - # Found text in line, do the highlight highlight. if find_index != -1: new_text = self.search_highlight_object(text, find_index, search_string) text_objects[i] = new_text self.state.add_flow_setting(self.flow, "last_search_index", find_index) + self.state.add_flow_setting(self.flow, "last_find_line", i) found = True @@ -409,7 +412,8 @@ class FlowView(common.WWrap): else: self.state.add_flow_setting(self.flow, "last_search_index", 0) self.state.add_flow_setting(self.flow, "last_find_line", 0) - text_objects, focus_pos = self.search_highlight_text(text_objects, search_string, True) + text_objects, focus_pos = self.search_highlight_text(text_objects, + search_string, looping=True, backwards=backwards) return text_objects, focus_pos diff --git a/test/test_console_search.py b/test/test_console_search.py index 69c033b9..60b998cc 100644 --- a/test/test_console_search.py +++ b/test/test_console_search.py @@ -38,8 +38,8 @@ def test_search_highlights_clears_prev(): text_object = tutils.get_body_line(f.last_displayed_body, 0) assert text_object.get_text() != ('this is string', [(None, 8), (f.highlight_color, 6)]) -def test_search_highlights_multi_line(): - f = tutils.tflowview(request_contents="this is string\nstring is string") +def test_search_highlights_multi_line(flow=None): + f = flow if flow else tutils.tflowview(request_contents="this is string\nstring is string") # should highlight the first line. f.search("string") @@ -54,7 +54,6 @@ def test_search_highlights_multi_line(): # should highlight third line, second appearance of string. f.search("string") text_object = tutils.get_body_line(f.last_displayed_body, 1) - print(text_object.get_text(), ('string is string', [(None, 10), (f.highlight_color, 6)])) assert text_object.get_text() == ('string is string', [(None, 10), (f.highlight_color, 6)]) def test_search_loops(): @@ -122,15 +121,38 @@ def test_search_backwards(): def test_search_back_multiline(): f = tutils.tflowview(request_contents="this is string\nstring is string") - f.search("string") + # shared assertions. highlight and pointers should now be on the third + # 'string' appearance + test_search_highlights_multi_line(f) + + # should highlight second line, first appearance of string. + f.search_again(backwards=True) + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert text_object.get_text() == ('string is string', [(None, 0), (f.highlight_color, 6)]) + + # should highlight the first line again. + f.search_again(backwards=True) text_object = tutils.get_body_line(f.last_displayed_body, 0) - first_match = ('this is string', [(None, 8), (f.highlight_color, 6)]) - assert text_object.get_text() == first_match + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) +def test_search_back_multi_multi_line(): + """ + same as above for some bugs the above won't catch. + """ + f = tutils.tflowview(request_contents="this is string\nthis is string\nthis is string") + + f.search("string") f.search_again() + f.search_again() + + # should be on second line + f.search_again(backwards=True) text_object = tutils.get_body_line(f.last_displayed_body, 1) - assert text_object.get_text() == ('string is string', [(None, 0), (f.highlight_color, 6)]) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + # first line now f.search_again(backwards=True) text_object = tutils.get_body_line(f.last_displayed_body, 0) - assert text_object.get_text() == first_match + print(text_object.get_text(), ('this is string', [(None, 8), (f.highlight_color, 6)])) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + -- cgit v1.2.3 From 9fe6b8fd268c95fc1a93148c6e2f4cd4d8a5bb05 Mon Sep 17 00:00:00 2001 From: Pedro Worcel Date: Sat, 22 Feb 2014 16:36:35 +1300 Subject: add docs --- libmproxy/console/flowview.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 624cf95d..990dc967 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -68,7 +68,8 @@ def _mkhelp(): ("space", "next flow"), ("|", "run script on this flow"), ("/", "search in response body (case sensitive)"), - ("n", "repeat previous search"), + ("n", "repeat search forward"), + ("N", "repeat search backwards"), ] text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) return text -- cgit v1.2.3 From 3e500344282a364f1fbd7245c49d980fe0bfab11 Mon Sep 17 00:00:00 2001 From: Pedro Worcel Date: Sat, 22 Feb 2014 17:15:37 +1300 Subject: fix the wrapping on backward searches --- libmproxy/console/flowview.py | 41 ++++++++++++++++++++++++++--------------- test/test_console_search.py | 20 +++++++++++++++++++- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 990dc967..8f4cb1eb 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -365,6 +365,19 @@ class FlowView(common.WWrap): return loop_range + def search_find(self, text, search_string, start_index, backwards): + if backwards == False: + find_index = text.find(search_string, start_index) + else: + if start_index != 0: + start_index -= len(search_string) + else: + start_index = None + + find_index = text.rfind(search_string, 0, start_index) + + return find_index + def search_highlight_text(self, text_objects, search_string, looping = False, backwards = False): start_line, start_index = self.search_get_start(search_string) i = start_line @@ -374,45 +387,43 @@ class FlowView(common.WWrap): loop_range = self.search_get_range(len(text_objects), start_line, backwards) for i in loop_range: text_object = text_objects[i] - if i != start_line: - start_index = 0 try: text, style = text_object.get_text() except AttributeError: raise SearchError() - if backwards == False: - find_index = text.find(search_string, start_index) - else: - if start_index != 0: - start_index -= len(search_string) - else: - start_index = None + if i != start_line: + start_index = 0 - find_index = text.rfind(search_string, 0, start_index) + find_index = self.search_find(text, search_string, start_index, backwards) if find_index != -1: new_text = self.search_highlight_object(text, find_index, search_string) text_objects[i] = new_text + found = True self.state.add_flow_setting(self.flow, "last_search_index", find_index) - self.state.add_flow_setting(self.flow, "last_find_line", i) - found = True break + # handle search WRAP if found: focus_pos = i else : - if (start_line == 0 and start_index == 0) or looping: + if looping: focus_pos = None else: - self.state.add_flow_setting(self.flow, "last_search_index", 0) - self.state.add_flow_setting(self.flow, "last_find_line", 0) + if not backwards: + self.state.add_flow_setting(self.flow, "last_search_index", 0) + self.state.add_flow_setting(self.flow, "last_find_line", 0) + else: + self.state.add_flow_setting(self.flow, "last_search_index", None) + self.state.add_flow_setting(self.flow, "last_find_line", len(text_objects) - 1) + text_objects, focus_pos = self.search_highlight_text(text_objects, search_string, looping=True, backwards=backwards) diff --git a/test/test_console_search.py b/test/test_console_search.py index 60b998cc..0e47ef79 100644 --- a/test/test_console_search.py +++ b/test/test_console_search.py @@ -153,6 +153,24 @@ def test_search_back_multi_multi_line(): # first line now f.search_again(backwards=True) text_object = tutils.get_body_line(f.last_displayed_body, 0) - print(text_object.get_text(), ('this is string', [(None, 8), (f.highlight_color, 6)])) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + +def test_search_backwards_wraps(): + """ + when searching past line 0, it should loop. + """ + f = tutils.tflowview(request_contents="this is string\nthis is string\nthis is string") + + # should be on second line + f.search("string") + f.search_again() + text_object = tutils.get_body_line(f.last_displayed_body, 1) + assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + + # should be on third now. + f.search_again(backwards=True) + message = f.search_again(backwards=True) + + text_object = tutils.get_body_line(f.last_displayed_body, 2) assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) -- cgit v1.2.3 From cffae49e348a444466b46ea08cd982e099cdabc3 Mon Sep 17 00:00:00 2001 From: Pedro Worcel Date: Sat, 22 Feb 2014 18:04:56 +1300 Subject: add helpful messages on wrap --- libmproxy/console/flowview.py | 36 +++++++++++++++++++++++++----------- test/test_console_search.py | 2 +- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 8f4cb1eb..f95f2ded 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -256,7 +256,7 @@ class FlowView(common.WWrap): ) return f - def search_wrapped_around(self, last_find_line, last_search_index): + def search_wrapped_around(self, last_find_line, last_search_index, backwards): """ returns true if search wrapped around the bottom. """ @@ -266,13 +266,22 @@ class FlowView(common.WWrap): current_search_index = self.state.get_flow_setting(self.flow, "last_search_index") - if current_find_line <= last_find_line: - return True - elif current_find_line == last_find_line: - if current_search_index <= last_search_index: - return True + if not backwards: + message = "search hit BOTTOM, continuing at TOP" + if current_find_line <= last_find_line: + return True, message + elif current_find_line == last_find_line: + if current_search_index <= last_search_index: + return True, message + else: + message = "search hit TOP, continuing at BOTTOM" + if current_find_line >= last_find_line: + return True, message + elif current_find_line == last_find_line: + if current_search_index >= last_search_index: + return True, message - return False + return False, "" def search_again(self, backwards=False): """ @@ -284,7 +293,10 @@ class FlowView(common.WWrap): if message: self.master.statusbar.message(message) else: - self.master.statusbar.message("no previous searches have been made") + message = "no previous searches have been made" + self.master.statusbar.message(message) + + return message def search(self, search_string, backwards=False): """ @@ -331,8 +343,11 @@ class FlowView(common.WWrap): self.last_displayed_body = list_box - if not backwards and self.search_wrapped_around(last_find_line, last_search_index): - return "search hit BOTTOM, continuing at TOP" + wrapped, wrapped_message = self.search_wrapped_around(last_find_line, last_search_index, backwards) + + if wrapped: + print(wrapped, wrapped_message) + return wrapped_message def search_get_start(self, search_string): start_line = 0 @@ -409,7 +424,6 @@ class FlowView(common.WWrap): break - # handle search WRAP if found: focus_pos = i diff --git a/test/test_console_search.py b/test/test_console_search.py index 0e47ef79..f4e9ff9b 100644 --- a/test/test_console_search.py +++ b/test/test_console_search.py @@ -170,7 +170,7 @@ def test_search_backwards_wraps(): # should be on third now. f.search_again(backwards=True) message = f.search_again(backwards=True) - text_object = tutils.get_body_line(f.last_displayed_body, 2) assert text_object.get_text() == ('this is string', [(None, 8), (f.highlight_color, 6)]) + assert message == "search hit TOP, continuing at BOTTOM" -- cgit v1.2.3 From a34a4831843d5745e200c1df58672973a57aebb0 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 2 Mar 2014 15:14:22 +1300 Subject: Adapt for new pathod and netlib APIs. --- libmproxy/console/flowview.py | 1 - libmproxy/proxy.py | 4 ++-- test/test_protocol_http.py | 3 --- test/test_server.py | 13 ++++++++++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index f95f2ded..3486f57e 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -346,7 +346,6 @@ class FlowView(common.WWrap): wrapped, wrapped_message = self.search_wrapped_around(last_find_line, last_search_index, backwards) if wrapped: - print(wrapped, wrapped_message) return wrapped_message def search_get_start(self, search_string): diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index b6480822..0203ba86 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -48,7 +48,7 @@ class ProxyConfig: self.forward_proxy = forward_proxy self.transparent_proxy = transparent_proxy self.authenticator = authenticator - self.certstore = certutils.CertStore() + self.certstore = certutils.CertStore(cacert) class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject): @@ -422,7 +422,7 @@ class ConnectionHandler: host = upstream_cert.cn.decode("utf8").encode("idna") sans = upstream_cert.altnames - ret = self.config.certstore.get_cert(host, sans, self.config.cacert) + ret = self.config.certstore.get_cert(host, sans) if not ret: raise ProxyError(502, "Unable to generate dummy cert.") return ret diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index d2ba24de..18aad3b6 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -88,7 +88,6 @@ class TestHTTPResponse: class TestInvalidRequests(tservers.HTTPProxTest): ssl = True - def test_double_connect(self): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) @@ -117,9 +116,7 @@ class TestProxyChaining(tservers.HTTPChainProxyTest): class TestProxyChainingSSL(tservers.HTTPChainProxyTest): ssl = True - def test_simple(self): - p = self.pathoc() req = p.request("get:'/p/418:b\"content\"'") assert req.content == "content" diff --git a/test/test_server.py b/test/test_server.py index 2f9e6728..a86c095c 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -206,13 +206,20 @@ class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin): def test_certfile(self): assert self.pathod("304") -class TestHTTPSNoCommonName(tservers.HTTPProxTest, CommonMixin): + +class TestHTTPSNoCommonName(tservers.HTTPProxTest): """ Test what happens if we get a cert without common name back. """ ssl = True - ssloptions=pathod.SSLOptions(certfile=tutils.test_data.path("data/no_common_name.pem"), - keyfile=tutils.test_data.path("data/no_common_name.pem")) + ssloptions=pathod.SSLOptions( + certfile = tutils.test_data.path("data/no_common_name.pem"), + cacert = tutils.test_data.path("data/no_common_name.pem"), + ) + def test_http(self): + f = self.pathod("202") + assert f.sslinfo.certchain[0].get_subject().CN == "127.0.0.1" + class TestReverse(tservers.ReverseProxTest, CommonMixin): reverse = True -- cgit v1.2.3 From 863b1e14552f5216ae4c47bf6dfe9b68ff2ca13b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 2 Mar 2014 15:58:53 +1300 Subject: Update for pathod.SSLOptions changes. --- test/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_server.py b/test/test_server.py index a86c095c..e31fb589 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -214,7 +214,7 @@ class TestHTTPSNoCommonName(tservers.HTTPProxTest): ssl = True ssloptions=pathod.SSLOptions( certfile = tutils.test_data.path("data/no_common_name.pem"), - cacert = tutils.test_data.path("data/no_common_name.pem"), + keyfile = tutils.test_data.path("data/no_common_name.pem"), ) def test_http(self): f = self.pathod("202") -- cgit v1.2.3 From f373ac5b6c443d0e633323e39b846fbe78822c2c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 2 Mar 2014 17:27:24 +1300 Subject: Improve explicit certificate specification - Support cert/key in the same PEM file - Rationalize arguments, expand tests, clean up a bit --- libmproxy/cmdline.py | 2 +- libmproxy/proxy.py | 40 ++++++++++++++++++++++++++++------------ test/test_proxy.py | 9 ++++++--- test/tservers.py | 1 + 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 8e7ab4a1..7950d40b 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -387,4 +387,4 @@ def common_options(parser): help="Allow access to users specified in an Apache htpasswd file." ) - proxy.certificate_option_group(parser) + proxy.ssl_option_group(parser) diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 0203ba86..9ff8887d 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -4,6 +4,7 @@ from netlib import tcp, http, certutils, http_auth import utils, version, platform, controller, stateobject TRANSPARENT_SSL_PORTS = [443, 8443] +CA_CERT_NAME = "mitmproxy-ca.pem" class AddressPriority(object): @@ -37,9 +38,10 @@ class Log: class ProxyConfig: - def __init__(self, certfile=None, cacert=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None, + def __init__(self, certfile=None, keyfile=None, cacert=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None): self.certfile = certfile + self.keyfile = keyfile self.cacert = cacert self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert @@ -381,7 +383,7 @@ class ConnectionHandler: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") dummycert = self.find_cert() - self.client_conn.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, + self.client_conn.convert_to_ssl(dummycert, self.config.keyfile or self.config.cacert, handle_sni=self.handle_sni) def server_reconnect(self, no_ssl=False): @@ -498,12 +500,17 @@ class DummyServer: # Command-line utils -def certificate_option_group(parser): +def ssl_option_group(parser): group = parser.add_argument_group("SSL") group.add_argument( - "--cert", action="store", - type=str, dest="cert", default=None, - help="User-created SSL certificate file." + "--certfile", action="store", + type=str, dest="certfile", default=None, + help="SSL certificate in PEM format, optionally with the key in the same file." + ) + group.add_argument( + "--keyfile", action="store", + type=str, dest="keyfile", default=None, + help="Key matching certfile." ) group.add_argument( "--client-certs", action="store", @@ -513,12 +520,20 @@ def certificate_option_group(parser): def process_proxy_options(parser, options): - if options.cert: - options.cert = os.path.expanduser(options.cert) - if not os.path.exists(options.cert): - return parser.error("Manually created certificate does not exist: %s" % options.cert) + if options.certfile: + options.certfile = os.path.expanduser(options.certfile) + if not os.path.exists(options.certfile): + return parser.error("Certificate file does not exist: %s" % options.certfile) + + if options.keyfile: + options.keyfile = os.path.expanduser(options.keyfile) + if not os.path.exists(options.keyfile): + return parser.error("Key file does not exist: %s" % options.keyfile) + + if options.certfile and not options.keyfile: + options.keyfile = options.certfile - cacert = os.path.join(options.confdir, "mitmproxy-ca.pem") + cacert = os.path.join(options.confdir, CA_CERT_NAME) cacert = os.path.expanduser(cacert) if not os.path.exists(cacert): certutils.dummy_ca(cacert) @@ -575,7 +590,8 @@ def process_proxy_options(parser, options): authenticator = http_auth.NullProxyAuth(None) return ProxyConfig( - certfile=options.cert, + certfile=options.certfile, + keyfile=options.keyfile, cacert=cacert, clientcerts=options.clientcerts, body_size_limit=body_size_limit, diff --git a/test/test_proxy.py b/test/test_proxy.py index c42d66e7..5ff00290 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -70,9 +70,12 @@ class TestProcessProxyOptions: def test_simple(self): assert self.p() - def test_cert(self): - self.assert_noerr("--cert", tutils.test_data.path("data/testkey.pem")) - self.assert_err("does not exist", "--cert", "nonexistent") + def test_certfile_keyfile(self): + self.assert_noerr("--certfile", tutils.test_data.path("data/testkey.pem")) + self.assert_err("does not exist", "--certfile", "nonexistent") + + self.assert_noerr("--keyfile", tutils.test_data.path("data/testkey.pem")) + self.assert_err("does not exist", "--keyfile", "nonexistent") def test_confdir(self): with tutils.tmpdir() as confdir: diff --git a/test/tservers.py b/test/tservers.py index 812e8921..a0f37c98 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -128,6 +128,7 @@ class ProxTestBase(object): d["clientcerts"] = tutils.test_data.path("data/clientcert") if cls.certfile: d["certfile"] =tutils.test_data.path("data/testkey.pem") + d["keyfile"] =tutils.test_data.path("data/testkey.pem") return d -- cgit v1.2.3 From 875f5f8cb65a254c40816e7cda0e4be96384ac16 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 2 Mar 2014 17:35:41 +1300 Subject: Cipher specification. --- libmproxy/proxy.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 9ff8887d..b787386a 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -38,8 +38,12 @@ class Log: class ProxyConfig: - def __init__(self, certfile=None, keyfile=None, cacert=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None, - reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None): + def __init__(self, certfile=None, keyfile=None, cacert=None, clientcerts=None, + no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, + forward_proxy=None, transparent_proxy=None, authenticator=None, + ciphers=None + ): + self.ciphers = ciphers self.certfile = certfile self.keyfile = keyfile self.cacert = cacert @@ -383,8 +387,13 @@ class ConnectionHandler: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") dummycert = self.find_cert() - self.client_conn.convert_to_ssl(dummycert, self.config.keyfile or self.config.cacert, - handle_sni=self.handle_sni) + print self.config.ciphers + self.client_conn.convert_to_ssl( + dummycert, + self.config.keyfile or self.config.cacert, + handle_sni = self.handle_sni, + cipher_list = self.config.ciphers + ) def server_reconnect(self, no_ssl=False): address = self.server_conn.address @@ -517,6 +526,11 @@ def ssl_option_group(parser): type=str, dest="clientcerts", default=None, help="Client certificate directory." ) + group.add_argument( + "--ciphers", action="store", + type=str, dest="ciphers", default=None, + help="SSL cipher specification." + ) def process_proxy_options(parser, options): @@ -599,5 +613,6 @@ def process_proxy_options(parser, options): reverse_proxy=rp, forward_proxy=fp, transparent_proxy=trans, - authenticator=authenticator + authenticator=authenticator, + ciphers=options.ciphers, ) -- cgit v1.2.3 From 32af66881465ae98a53665c8ddd42c02aaf492f7 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 2 Mar 2014 22:15:53 +1300 Subject: Minor cleanups. --- libmproxy/protocol/__init__.py | 5 +++-- libmproxy/protocol/tcp.py | 13 ++++++------- libmproxy/proxy.py | 2 -- setup.py | 2 +- test/test_server.py | 3 --- test/tservers.py | 4 +++- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index 580d693c..2c2e7285 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -35,7 +35,6 @@ class TemporaryServerChangeMixin(object): This mixin allows safe modification of the target server, without any need to expose the ConnectionHandler to the Flow. """ - def change_server(self, address, ssl): if address == self.c.server_conn.address(): return @@ -98,4 +97,6 @@ def handle_messages(conntype, connection_handler): def handle_error(conntype, connection_handler, error): - return _handler(conntype, connection_handler).handle_error(error) \ No newline at end of file + return _handler(conntype, connection_handler).handle_error(error) + + diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index df7e6692..1591eb04 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -32,12 +32,11 @@ class TCPHandler(ProtocolHandler): if d == "": # connection closed break data.write(d) - - """ - OpenSSL Connections have an internal buffer that might contain data altough everything is read - from the socket. Thankfully, connection.pending() returns the amount of bytes in this buffer, - so we can read it completely at once. - """ + # OpenSSL Connections have an internal buffer that might + # contain data altough everything is read from the socket. + # Thankfully, connection.pending() returns the amount of + # bytes in this buffer, so we can read it completely at + # once. if src.ssl_established: data.write(rfile.read(src.connection.pending())) else: # no data left, but not closed yet @@ -57,4 +56,4 @@ class TCPHandler(ProtocolHandler): self.c.log("%s %s\r\n%s" % (direction, dst_str,data)) dst.wfile.write(data) - dst.wfile.flush() \ No newline at end of file + dst.wfile.flush() diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index b787386a..b0fb9449 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -387,7 +387,6 @@ class ConnectionHandler: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") dummycert = self.find_cert() - print self.config.ciphers self.client_conn.convert_to_ssl( dummycert, self.config.keyfile or self.config.cacert, @@ -469,7 +468,6 @@ class ProxyServerError(Exception): class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True - def __init__(self, config, port, host='', server_version=version.NAMEVERSION): """ Raises ProxyServerError if there's a startup problem. diff --git a/setup.py b/setup.py index d8e25660..624a1a60 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ setup( "netlib>=%s"%version.VERSION, "urwid>=1.1", "pyasn1>0.1.2", - "pyopenssl>=0.13", + "pyopenssl>=0.14", "Pillow>=2.3.0", "lxml", "flask" diff --git a/test/test_server.py b/test/test_server.py index e31fb589..2714ef52 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -377,7 +377,6 @@ class TestTransparentResolveError(tservers.TransparentProxTest): assert self.pathod("304").status_code == 502 - class MasterIncomplete(tservers.TestMaster): def handle_request(self, m): resp = tutils.tresp() @@ -390,5 +389,3 @@ class TestIncompleteResponse(tservers.HTTPProxTest): def test_incomplete(self): assert self.pathod("200").status_code == 502 - - diff --git a/test/tservers.py b/test/tservers.py index a0f37c98..540cda60 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -28,7 +28,6 @@ class TestMaster(flow.FlowMaster): self.apps.add(testapp, "testapp", 80) self.apps.add(errapp, "errapp", 80) self.clear_log() - self.start_app(APP_HOST, APP_PORT, False) def handle_request(self, m): flow.FlowMaster.handle_request(self, m) @@ -77,6 +76,7 @@ class ProxTestBase(object): no_upstream_cert = False authenticator = None masterclass = TestMaster + externalapp = False @classmethod def setupAll(cls): cls.server = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) @@ -89,6 +89,7 @@ class ProxTestBase(object): **pconf ) tmaster = cls.masterclass(config) + tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.proxy = ProxyThread(tmaster) cls.proxy.start() @@ -266,6 +267,7 @@ class ChainProxTest(ProxTestBase): cls.chain[-1].port ) tmaster = cls.masterclass(config) + tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.chain.append(ProxyThread(tmaster)) cls.chain[-1].start() -- cgit v1.2.3 From d65f2215cb9191a24b36ad6a4fcbf474798d3b2d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 5 Mar 2014 17:25:12 +1300 Subject: Much more sophisticated cert handling - Specify per-domain certificates and keys - Certs are no longer regenerated for SANs - And more. :) --- libmproxy/app.py | 7 ++- libmproxy/proxy.py | 98 +++++++++++++++++----------------------- test/test_app.py | 4 +- test/test_console_contentview.py | 2 +- test/test_proxy.py | 14 +++--- test/test_server.py | 5 +- test/tservers.py | 13 +++--- 7 files changed, 63 insertions(+), 80 deletions(-) diff --git a/libmproxy/app.py b/libmproxy/app.py index b046f712..3ebbb61b 100644 --- a/libmproxy/app.py +++ b/libmproxy/app.py @@ -1,5 +1,6 @@ import flask import os.path +import proxy mapp = flask.Flask(__name__) mapp.debug = True @@ -16,14 +17,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - capath = master().server.config.cacert - p = os.path.splitext(capath)[0] + "-cert.pem" + p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - capath = master().server.config.cacert - p = os.path.splitext(capath)[0] + "-cert.p12" + p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index b0fb9449..1eebba07 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -4,9 +4,12 @@ from netlib import tcp, http, certutils, http_auth import utils, version, platform, controller, stateobject TRANSPARENT_SSL_PORTS = [443, 8443] +CONF_BASENAME = "mitmproxy" +CONF_DIR = "~/.mitmproxy" CA_CERT_NAME = "mitmproxy-ca.pem" + class AddressPriority(object): """ Enum that signifies the priority of the given address when choosing the destination host. @@ -38,15 +41,12 @@ class Log: class ProxyConfig: - def __init__(self, certfile=None, keyfile=None, cacert=None, clientcerts=None, + def __init__(self, confdir=CONF_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None, - ciphers=None + ciphers=None, certs=None ): self.ciphers = ciphers - self.certfile = certfile - self.keyfile = keyfile - self.cacert = cacert self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit @@ -54,7 +54,9 @@ class ProxyConfig: self.forward_proxy = forward_proxy self.transparent_proxy = transparent_proxy self.authenticator = authenticator - self.certstore = certutils.CertStore(cacert) + self.confdir = os.path.expanduser(confdir) + self.certstore = certutils.CertStore.from_store(confdir, CONF_BASENAME) + class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject): @@ -386,10 +388,9 @@ class ConnectionHandler: if client: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") - dummycert = self.find_cert() + cert, key = self.find_cert() self.client_conn.convert_to_ssl( - dummycert, - self.config.keyfile or self.config.cacert, + cert, key, handle_sni = self.handle_sni, cipher_list = self.config.ciphers ) @@ -420,22 +421,18 @@ class ConnectionHandler: self.channel.tell("log", Log(msg)) def find_cert(self): - if self.config.certfile: - with open(self.config.certfile, "rb") as f: - return certutils.SSLCert.from_pem(f.read()) - else: - host = self.server_conn.address.host - sans = [] - if not self.config.no_upstream_cert or not self.server_conn.ssl_established: - upstream_cert = self.server_conn.cert - if upstream_cert.cn: - host = upstream_cert.cn.decode("utf8").encode("idna") - sans = upstream_cert.altnames - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret + host = self.server_conn.address.host + sans = [] + if not self.config.no_upstream_cert or not self.server_conn.ssl_established: + upstream_cert = self.server_conn.cert + if upstream_cert.cn: + host = upstream_cert.cn.decode("utf8").encode("idna") + sans = upstream_cert.altnames + + ret = self.config.certstore.get_cert(host, sans) + if not ret: + raise ProxyError(502, "Unable to generate dummy cert.") + return ret def handle_sni(self, connection): """ @@ -451,9 +448,9 @@ class ConnectionHandler: self.server_reconnect() # reconnect to upstream server with SNI # Now, change client context to reflect changed certificate: new_context = SSL.Context(SSL.TLSv1_METHOD) - new_context.use_privatekey_file(self.config.certfile or self.config.cacert) - dummycert = self.find_cert() - new_context.use_certificate(dummycert.x509) + cert, key = self.find_cert() + new_context.use_privatekey_file(key) + new_context.use_certificate(cert.X509) connection.set_context(new_context) # An unhandled exception in this method will core dump PyOpenSSL, so # make dang sure it doesn't happen. @@ -510,14 +507,12 @@ class DummyServer: def ssl_option_group(parser): group = parser.add_argument_group("SSL") group.add_argument( - "--certfile", action="store", - type=str, dest="certfile", default=None, - help="SSL certificate in PEM format, optionally with the key in the same file." - ) - group.add_argument( - "--keyfile", action="store", - type=str, dest="keyfile", default=None, - help="Key matching certfile." + "--cert", dest='certs', default=[], type=str, + metavar = "SPEC", action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". '\ + 'The domain may include a wildcard, and is equal to "*" if not specified. '\ + 'The file at path is a certificate in PEM format. If a private key is included in the PEM, '\ + 'it is used, else the default key in the conf dir is used. Can be passed multiple times.' ) group.add_argument( "--client-certs", action="store", @@ -532,23 +527,6 @@ def ssl_option_group(parser): def process_proxy_options(parser, options): - if options.certfile: - options.certfile = os.path.expanduser(options.certfile) - if not os.path.exists(options.certfile): - return parser.error("Certificate file does not exist: %s" % options.certfile) - - if options.keyfile: - options.keyfile = os.path.expanduser(options.keyfile) - if not os.path.exists(options.keyfile): - return parser.error("Key file does not exist: %s" % options.keyfile) - - if options.certfile and not options.keyfile: - options.keyfile = options.certfile - - cacert = os.path.join(options.confdir, CA_CERT_NAME) - cacert = os.path.expanduser(cacert) - if not os.path.exists(cacert): - certutils.dummy_ca(cacert) body_size_limit = utils.parse_size(options.body_size_limit) if options.reverse_proxy and options.transparent_proxy: return parser.error("Can't set both reverse proxy and transparent proxy.") @@ -601,10 +579,17 @@ def process_proxy_options(parser, options): else: authenticator = http_auth.NullProxyAuth(None) + certs = [] + for i in options.certs: + parts = i.split("=", 1) + if len(parts) == 1: + parts = ["*", parts[0]] + parts[1] = os.path.expanduser(parts[1]) + if not os.path.exists(parts[1]): + parser.error("Certificate file does not exist: %s"%parts[1]) + certs.append(parts) + return ProxyConfig( - certfile=options.certfile, - keyfile=options.keyfile, - cacert=cacert, clientcerts=options.clientcerts, body_size_limit=body_size_limit, no_upstream_cert=options.no_upstream_cert, @@ -613,4 +598,5 @@ def process_proxy_options(parser, options): transparent_proxy=trans, authenticator=authenticator, ciphers=options.ciphers, + certs = certs, ) diff --git a/test/test_app.py b/test/test_app.py index f0eab7cc..52cd1ba6 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -9,11 +9,9 @@ class TestApp(tservers.HTTPProxTest): assert self.app("/").status_code == 200 def test_cert(self): - path = tutils.test_data.path("data/confdir/") + "mitmproxy-ca-cert." with tutils.tmpdir() as d: for ext in ["pem", "p12"]: resp = self.app("/cert/%s" % ext) assert resp.status_code == 200 - with open(path + ext, "rb") as f: - assert resp.content == f.read() + assert resp.content diff --git a/test/test_console_contentview.py b/test/test_console_contentview.py index a878ad4e..0aabd2c5 100644 --- a/test/test_console_contentview.py +++ b/test/test_console_contentview.py @@ -120,7 +120,7 @@ class TestContentView: def test_view_css(self): v = cv.ViewCSS() - with open('./test/data/1.css', 'r') as fp: + with open(tutils.test_data.path('data/1.css'), 'r') as fp: fixture_1 = fp.read() result = v([], 'a', 100) diff --git a/test/test_proxy.py b/test/test_proxy.py index 5ff00290..b15e3f84 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -70,13 +70,6 @@ class TestProcessProxyOptions: def test_simple(self): assert self.p() - def test_certfile_keyfile(self): - self.assert_noerr("--certfile", tutils.test_data.path("data/testkey.pem")) - self.assert_err("does not exist", "--certfile", "nonexistent") - - self.assert_noerr("--keyfile", tutils.test_data.path("data/testkey.pem")) - self.assert_err("does not exist", "--keyfile", "nonexistent") - def test_confdir(self): with tutils.tmpdir() as confdir: self.assert_noerr("--confdir", confdir) @@ -93,11 +86,16 @@ class TestProcessProxyOptions: self.assert_err("invalid reverse proxy", "-P", "reverse") self.assert_noerr("-P", "http://localhost") - def test_certs(self): + def test_client_certs(self): with tutils.tmpdir() as confdir: self.assert_noerr("--client-certs", confdir) self.assert_err("directory does not exist", "--client-certs", "nonexistent") + def test_certs(self): + with tutils.tmpdir() as confdir: + self.assert_noerr("--cert", tutils.test_data.path("data/testkey.pem")) + self.assert_err("does not exist", "--cert", "nonexistent") + def test_auth(self): p = self.assert_noerr("--nonanonymous") assert p.authenticator diff --git a/test/test_server.py b/test/test_server.py index 2714ef52..ed21e75c 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -213,8 +213,9 @@ class TestHTTPSNoCommonName(tservers.HTTPProxTest): """ ssl = True ssloptions=pathod.SSLOptions( - certfile = tutils.test_data.path("data/no_common_name.pem"), - keyfile = tutils.test_data.path("data/no_common_name.pem"), + certs = [ + ("*", tutils.test_data.path("data/no_common_name.pem")) + ] ) def test_http(self): f = self.pathod("202") diff --git a/test/tservers.py b/test/tservers.py index 540cda60..3a6a610f 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -1,4 +1,5 @@ import threading, Queue +import shutil, tempfile import flask import libpathod.test, libpathod.pathoc from libmproxy import proxy, flow, controller @@ -72,7 +73,6 @@ class ProxTestBase(object): ssl = None ssloptions = False clientcerts = False - certfile = None no_upstream_cert = False authenticator = None masterclass = TestMaster @@ -82,9 +82,10 @@ class ProxTestBase(object): cls.server = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) cls.server2 = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) pconf = cls.get_proxy_config() + cls.confdir = tempfile.gettempdir() config = proxy.ProxyConfig( no_upstream_cert = cls.no_upstream_cert, - cacert = tutils.test_data.path("data/confdir/mitmproxy-ca.pem"), + confdir = cls.confdir, authenticator = cls.authenticator, **pconf ) @@ -93,6 +94,10 @@ class ProxTestBase(object): cls.proxy = ProxyThread(tmaster) cls.proxy.start() + @classmethod + def tearDownAll(cls): + shutil.rmtree(cls.confdir) + @property def master(cls): return cls.proxy.tmaster @@ -127,9 +132,6 @@ class ProxTestBase(object): d = dict() if cls.clientcerts: d["clientcerts"] = tutils.test_data.path("data/clientcert") - if cls.certfile: - d["certfile"] =tutils.test_data.path("data/testkey.pem") - d["keyfile"] =tutils.test_data.path("data/testkey.pem") return d @@ -254,7 +256,6 @@ class ChainProxTest(ProxTestBase): """ n = 2 chain_config = [lambda: proxy.ProxyConfig( - cacert = tutils.test_data.path("data/confdir/mitmproxy-ca.pem"), )] * n @classmethod def setupAll(cls): -- cgit v1.2.3 From edac95028a9e0371c4c22069f2a2603046bc3694 Mon Sep 17 00:00:00 2001 From: Sergey Chipiga Date: Wed, 5 Mar 2014 21:00:29 +0400 Subject: remove redundant codeline --- libmproxy/console/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 4f298698..4a58e771 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -77,7 +77,6 @@ class PathEdit(urwid.Edit, _PathCompleter): class ActionBar(common.WWrap): def __init__(self): self.message("") - self.expire = None def selectable(self): return True -- cgit v1.2.3 From d0e6fa270533689fdaca0daab2d9dfa209bdfc26 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 7 Mar 2014 15:20:15 +1300 Subject: Use the right conf dir... --- libmproxy/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 1eebba07..6dd37752 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -55,7 +55,7 @@ class ProxyConfig: self.transparent_proxy = transparent_proxy self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) - self.certstore = certutils.CertStore.from_store(confdir, CONF_BASENAME) + self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) -- cgit v1.2.3 From 221973aff6553ffb1d40859c5d35b88d959f9718 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 7 Mar 2014 16:38:24 +1300 Subject: Revert to old cert names, use a subdir for test conf dir. --- libmproxy/app.py | 6 +++--- test/test_app.py | 1 - test/tservers.py | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libmproxy/app.py b/libmproxy/app.py index 3ebbb61b..24187704 100644 --- a/libmproxy/app.py +++ b/libmproxy/app.py @@ -1,5 +1,5 @@ import flask -import os.path +import os.path, os import proxy mapp = flask.Flask(__name__) @@ -17,12 +17,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-cert.pem") + p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-ca-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-cert.p12") + p = os.path.join(master().server.config.confdir, proxy.CONF_BASENAME + "-ca-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/test/test_app.py b/test/test_app.py index 52cd1ba6..0b6ed14c 100644 --- a/test/test_app.py +++ b/test/test_app.py @@ -14,4 +14,3 @@ class TestApp(tservers.HTTPProxTest): resp = self.app("/cert/%s" % ext) assert resp.status_code == 200 assert resp.content - diff --git a/test/tservers.py b/test/tservers.py index 3a6a610f..cf9b3f73 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -1,3 +1,4 @@ +import os.path import threading, Queue import shutil, tempfile import flask @@ -82,7 +83,7 @@ class ProxTestBase(object): cls.server = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) cls.server2 = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) pconf = cls.get_proxy_config() - cls.confdir = tempfile.gettempdir() + cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") config = proxy.ProxyConfig( no_upstream_cert = cls.no_upstream_cert, confdir = cls.confdir, -- cgit v1.2.3