1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
import copy
import logging
import typing
from typing import Dict
from bs4 import BeautifulSoup
from mitmproxy.http import HTTPFlow
from examples.complex.webscanner_helper.urldict import URLDict
NO_CONTENT = object()
class MappingAddonConfig:
HTML_PARSER = "html.parser"
class MappingAddon:
""" The mapping add-on can be used in combination with web application scanners to reduce their false positives.
Many web application scanners produce false positives caused by dynamically changing content of web applications
such as the current time or current measurements. When testing for injection vulnerabilities, web application
scanners are tricked into thinking they changed the content with the injected payload. In realty, the content of
the web application changed notwithstanding the scanner's input. When the mapping add-on is used to map the content
to a fixed value, these false positives can be avoided.
"""
OPT_MAPPING_FILE = "mapping_file"
"""File where urls and css selector to mapped content is stored.
Elements will be replaced with the content given in this file. If the content is none it will be set to the first
seen value.
Example:
{
"http://10.10.10.10": {
"body": "My Text"
},
"URL": {
"css selector": "Replace with this"
}
}
"""
OPT_MAP_PERSISTENT = "map_persistent"
"""Whether to store all new content in the configuration file."""
def __init__(self, filename: str, persistent: bool = False) -> None:
""" Initializes the mapping add-on
Args:
filename: str that provides the name of the file in which the urls and css selectors to mapped content is
stored.
persistent: bool that indicates whether to store all new content in the configuration file.
Example:
The file in which the mapping config is given should be in the following format:
{
"http://10.10.10.10": {
"body": "My Text"
},
"<URL>": {
"<css selector>": "Replace with this"
}
}
"""
self.filename = filename
self.persistent = persistent
self.logger = logging.getLogger(self.__class__.__name__)
with open(filename, "r") as f:
self.mapping_templates = URLDict.load(f)
def load(self, loader):
loader.add_option(
self.OPT_MAPPING_FILE, str, "",
"File where replacement configuration is stored."
)
loader.add_option(
self.OPT_MAP_PERSISTENT, bool, False,
"Whether to store all new content in the configuration file."
)
def configure(self, updated):
if self.OPT_MAPPING_FILE in updated:
self.filename = updated[self.OPT_MAPPING_FILE]
with open(self.filename, "r") as f:
self.mapping_templates = URLDict.load(f)
if self.OPT_MAP_PERSISTENT in updated:
self.persistent = updated[self.OPT_MAP_PERSISTENT]
def replace(self, soup: BeautifulSoup, css_sel: str, replace: BeautifulSoup) -> None:
"""Replaces the content of soup that matches the css selector with the given replace content."""
for content in soup.select(css_sel):
self.logger.debug(f"replace \"{content}\" with \"{replace}\"")
content.replace_with(copy.copy(replace))
def apply_template(self, soup: BeautifulSoup, template: Dict[str, typing.Union[BeautifulSoup]]) -> None:
"""Applies the given mapping template to the given soup."""
for css_sel, replace in template.items():
mapped = soup.select(css_sel)
if not mapped:
self.logger.warning(f"Could not find \"{css_sel}\", can not freeze anything.")
else:
self.replace(soup, css_sel, BeautifulSoup(replace, features=MappingAddonConfig.HTML_PARSER))
def response(self, flow: HTTPFlow) -> None:
"""If a response is received, check if we should replace some content. """
try:
templates = self.mapping_templates[flow]
res = flow.response
if res is not None:
encoding = res.headers.get("content-encoding", "utf-8")
content_type = res.headers.get("content-type", "text/html")
if "text/html" in content_type and encoding == "utf-8":
content = BeautifulSoup(res.content, MappingAddonConfig.HTML_PARSER)
for template in templates:
self.apply_template(content, template)
res.content = content.encode(encoding)
else:
self.logger.warning(f"Unsupported content type '{content_type}' or content encoding '{encoding}'")
except KeyError:
pass
def done(self) -> None:
"""Dumps all new content into the configuration file if self.persistent is set."""
if self.persistent:
# make sure that all items are strings and not soups.
def value_dumper(value):
store = {}
if value is None:
return "None"
try:
for css_sel, soup in value.items():
store[css_sel] = str(soup)
except:
raise RuntimeError(value)
return store
with open(self.filename, "w") as f:
self.mapping_templates.dump(f, value_dumper)
|
