From 95d725cda96b86ca9ad81924cbcac47c857a2bd2 Mon Sep 17 00:00:00 2001 From: kimbo Date: Wed, 4 Mar 2020 21:16:02 -0700 Subject: example for blocking DNS queries over HTTPS --- examples/complex/block_dns_over_https.py | 234 +++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 examples/complex/block_dns_over_https.py (limited to 'examples') diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py new file mode 100644 index 00000000..a5e03a43 --- /dev/null +++ b/examples/complex/block_dns_over_https.py @@ -0,0 +1,234 @@ +""" +This module is for blocking DNS over HTTPS requests. + +It loads a blocklist of IPs and hostnames that are known to serve DNS over HTTPS requests. +It also uses headers, query params, and paths to detect DoH (and block it) +""" +import json +import re +import os +import urllib.request + +import dns.query +import dns.rdatatype +import dns.message +import dns.resolver +import dns.rdtypes.IN.A +import dns.rdtypes.IN.AAAA + +from mitmproxy import ctx + +# filename we'll save the blocklist to so we don't have to re-generate it every time +blocklist_filename = 'blocklist.json' + +# additional hostnames to block +additional_doh_names = [ + 'dns.google.com' +] + +# additional IPs to block +additional_doh_ips = [ + +] + +def get_doh_providers(): + """ + Scrape a list of DoH providers from curl's wiki page. + :return: a generator of dicts containing information about the DoH providers + """ + https_url_re = re.compile(r'https://' + r'(?P[0-9a-zA-Z._~-]+)' + r'(?P:[0-9]+)?' + r'(?P[0-9a-zA-Z._~/-]+)?') + + provider_re = re.compile(r'(\[([^\]]+)\]\(([^)]+))\)|(.*)') + # URLs that are not DoH URLs + do_not_include = ['my.nextdns.io', 'blog.cloudflare.com'] + found_table = False + with urllib.request.urlopen('https://raw.githubusercontent.com/wiki/curl/curl/DNS-over-HTTPS.md') as fp: + for line in fp: + line = line.decode() + if line.startswith('|'): + if not found_table: + found_table = True + continue + cols = line.split('|') + provider_col = cols[1].strip() + website = None + provider_name = None + matches = provider_re.findall(provider_col) + if matches[0][3] != '': + provider_name = matches[0][3] + if matches[0][1] != '': + provider_name = matches[0][1] + if matches[0][2] != '': + website = matches[0][2] + if provider_name is not None: + provider_name = re.sub(r'([^[]+)\s?(.*)', r'\1', provider_name) + while provider_name[-1] == ' ': + provider_name = provider_name[:-1] + url_col = cols[2] + doh_url_matches = https_url_re.findall(url_col) + if len(doh_url_matches) == 0: + continue + else: + for doh_url in doh_url_matches: + if doh_url[0] in do_not_include: + continue + yield { + 'name': provider_name, + 'website': website, + 'url': 'https://{}{}{}'.format(doh_url[0], ':{}'.format(doh_url[1]) if len(doh_url[1]) != 0 else '', doh_url[2]), + 'hostname': doh_url[0], + 'port': doh_url[1] if len(doh_url[1]) != 0 else '443', + 'path': doh_url[2], + } + if found_table and line.startswith('#'): + break + return + +def get_ips(hostname): + """ + Lookup all A and AAAA records for given hostname + :param hostname: the name to lookup + :return: a list of IP addresses returned + """ + default_nameserver = dns.resolver.Resolver().nameservers[0] + ips = list() + rdtypes = [dns.rdatatype.A, dns.rdatatype.AAAA] + for rdtype in rdtypes: + q = dns.message.make_query(hostname, rdtype) + r = dns.query.udp(q, default_nameserver) + if r.flags & dns.flags.TC: + r = dns.query.tcp(q, default_nameserver) + for a in r.answer: + for i in a.items: + if isinstance(i, dns.rdtypes.IN.A.A) or isinstance(i, dns.rdtypes.IN.AAAA.AAAA): + ips.append(str(i.address)) + return ips + +def load_blocklist(): + """ + Load a tuple containing two lists, in the form of (hostnames, ips). + It will attempt to load it from a file, and if that file is not found, + it will generate the blocklist and save it to a file. + + :return: a ``tuple`` of (``list``, ``list``), the hostnames and IPs to block + """ + if os.path.isfile(blocklist_filename): + with open(blocklist_filename, 'r') as fp: + j = json.load(fp) + doh_hostnames, doh_ips = j['hostnames'], j['ips'] + else: + doh_hostnames = list([i['hostname'] for i in get_doh_providers()]) + doh_ips = list() + for hostname in doh_hostnames: + ips = get_ips(hostname) + doh_ips.extend(ips) + doh_hostnames.extend(additional_doh_names) + doh_ips.extend(additional_doh_ips) + with open(blocklist_filename, 'w') as fp: + obj = { + 'hostnames': doh_hostnames, + 'ips': doh_ips + } + json.dump(obj, fp=fp) + return doh_hostnames, doh_ips + +# load DoH hostnames and IP addresses to block +doh_hostnames, doh_ips = load_blocklist() +ctx.log.info('DoH blocklist loaded') + +# convert to sets for faster lookups +doh_hostnames = set(doh_hostnames) +doh_ips = set(doh_ips) + + +def _has_dns_message_content_type(flow): + """ + Check if HTTP request has a DNS-looking 'Content-Type' header + + :param flow: mitmproxy flow + :return: True if 'Content-Type' header is DNS-looking, False otherwise + """ + doh_content_types = ['application/dns-message'] + if 'Content-Type' in flow.request.headers: + if flow.request.headers['Content-Type'] in doh_content_types: + return True + return False + +def _request_has_dns_query_string(flow): + """ + Check if the query string of a request contains the parameter 'dns' + + :param flow: mitmproxy flow + :return: True is 'dns' is a parameter in the query string, False otherwise + """ + return 'dns' in flow.request.query + +def _request_is_dns_json(flow): + """ + Check if the request looks like DoH with JSON. + + The only known implementations of DoH with JSON are Cloudflare and Google. + + For more info, see: + - https://developers.cloudflare.com/1.1.1.1/dns-over-https/json-format/ + - https://developers.google.com/speed/public-dns/docs/doh/json + + :param flow: mitmproxy flow + :return: True is request looks like DNS JSON, False otherwise + """ + # Header 'Accept: application/dns-json' is required in Cloudflare's DoH JSON API + # or they return a 400 HTTP response code + if 'Accept' in flow.request.headers: + if flow.request.headers['Accept'] == 'application/dns-json': + return True + # Google's DoH JSON API is https://dns.google/resolve + path = flow.request.path.split('?')[0] + if flow.request.host == 'dns.google' and path == '/resolve': + return True + return False + +def _request_has_doh_looking_path(flow): + """ + Check if the path looks like it's DoH. + Most common one is '/dns-query', likely because that's what's in the RFC + + :param flow: mitmproxy flow + :return: True if path looks like it's DoH, otherwise False + """ + doh_paths = [ + '/dns-query', # used in example in RFC 8484 (see https://tools.ietf.org/html/rfc8484#section-4.1.1) + ] + path = flow.request.path.split('?')[0] + return path in doh_paths + +def _requested_hostname_is_in_doh_blacklist(flow): + """ + Check if server hostname is in our DoH provider blacklist. + + The current blacklist is taken from https://github.com/curl/curl/wiki/DNS-over-HTTPS. + + :param flow: mitmproxy flow + :return: True if server's hostname is in DoH blacklist, otherwise False + """ + hostname = flow.request.host + ip = flow.server_conn.address + return hostname in doh_hostnames or hostname in doh_ips or ip in doh_ips + +doh_request_detection_checks = [ + _has_dns_message_content_type, + _request_has_dns_query_string, + _request_is_dns_json, + _requested_hostname_is_in_doh_blacklist, + _request_has_doh_looking_path +] + +def request(flow): + for check in doh_request_detection_checks: + is_doh = check(flow) + if is_doh: + ctx.log.warn("[DoH Detection] DNS over HTTPS request detected via method \"%s\"" % check.__name__) + flow.kill() + break -- cgit v1.2.3 From 81113a0dcc9d8cf2e2757c2dddc11df1f8dd3e14 Mon Sep 17 00:00:00 2001 From: kimbo Date: Wed, 4 Mar 2020 21:16:30 -0700 Subject: add block doh example to examples/complex/README --- examples/complex/README.md | 1 + 1 file changed, 1 insertion(+) (limited to 'examples') diff --git a/examples/complex/README.md b/examples/complex/README.md index c53503e4..923aadf1 100644 --- a/examples/complex/README.md +++ b/examples/complex/README.md @@ -2,6 +2,7 @@ | Filename | Description | |:-------------------------|:----------------------------------------------------------------------------------------------| +| block_dns_over_https.py | Use mitmproxy to block DNS over HTTPS (DoH) queries | | change_upstream_proxy.py | Dynamically change the upstream proxy. | | dns_spoofing.py | Use mitmproxy in a DNS spoofing scenario. | | dup_and_replay.py | Duplicates each request, changes it, and then replays the modified request. | -- cgit v1.2.3 From a70ab62797565bf29c5c80dcb83b2ff1e065d3fb Mon Sep 17 00:00:00 2001 From: kimbo Date: Wed, 4 Mar 2020 22:06:27 -0700 Subject: fix lint errors --- examples/complex/block_dns_over_https.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'examples') diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py index a5e03a43..864da20e 100644 --- a/examples/complex/block_dns_over_https.py +++ b/examples/complex/block_dns_over_https.py @@ -8,6 +8,7 @@ import json import re import os import urllib.request +from typing import List import dns.query import dns.rdatatype @@ -22,15 +23,16 @@ from mitmproxy import ctx blocklist_filename = 'blocklist.json' # additional hostnames to block -additional_doh_names = [ +additional_doh_names: List[str] = [ 'dns.google.com' ] # additional IPs to block -additional_doh_ips = [ +additional_doh_ips: List[str] = [ ] + def get_doh_providers(): """ Scrape a list of DoH providers from curl's wiki page. @@ -78,7 +80,10 @@ def get_doh_providers(): yield { 'name': provider_name, 'website': website, - 'url': 'https://{}{}{}'.format(doh_url[0], ':{}'.format(doh_url[1]) if len(doh_url[1]) != 0 else '', doh_url[2]), + 'url': 'https://{}{}{}'.format(doh_url[0], + ':{}'.format(doh_url[1]) + if len(doh_url[1]) != 0 + else '', doh_url[2]), 'hostname': doh_url[0], 'port': doh_url[1] if len(doh_url[1]) != 0 else '443', 'path': doh_url[2], @@ -87,6 +92,7 @@ def get_doh_providers(): break return + def get_ips(hostname): """ Lookup all A and AAAA records for given hostname @@ -107,6 +113,7 @@ def get_ips(hostname): ips.append(str(i.address)) return ips + def load_blocklist(): """ Load a tuple containing two lists, in the form of (hostnames, ips). @@ -135,6 +142,7 @@ def load_blocklist(): json.dump(obj, fp=fp) return doh_hostnames, doh_ips + # load DoH hostnames and IP addresses to block doh_hostnames, doh_ips = load_blocklist() ctx.log.info('DoH blocklist loaded') @@ -157,6 +165,7 @@ def _has_dns_message_content_type(flow): return True return False + def _request_has_dns_query_string(flow): """ Check if the query string of a request contains the parameter 'dns' @@ -166,6 +175,7 @@ def _request_has_dns_query_string(flow): """ return 'dns' in flow.request.query + def _request_is_dns_json(flow): """ Check if the request looks like DoH with JSON. @@ -190,6 +200,7 @@ def _request_is_dns_json(flow): return True return False + def _request_has_doh_looking_path(flow): """ Check if the path looks like it's DoH. @@ -204,6 +215,7 @@ def _request_has_doh_looking_path(flow): path = flow.request.path.split('?')[0] return path in doh_paths + def _requested_hostname_is_in_doh_blacklist(flow): """ Check if server hostname is in our DoH provider blacklist. @@ -217,6 +229,7 @@ def _requested_hostname_is_in_doh_blacklist(flow): ip = flow.server_conn.address return hostname in doh_hostnames or hostname in doh_ips or ip in doh_ips + doh_request_detection_checks = [ _has_dns_message_content_type, _request_has_dns_query_string, @@ -225,6 +238,7 @@ doh_request_detection_checks = [ _request_has_doh_looking_path ] + def request(flow): for check in doh_request_detection_checks: is_doh = check(flow) -- cgit v1.2.3 From f36a5b8aa8aaa3576a1ee962516120a9de0cd03c Mon Sep 17 00:00:00 2001 From: kimbo Date: Sat, 4 Apr 2020 21:18:58 -0600 Subject: replace scraping and DNS lookups with static list --- examples/complex/block_dns_over_https.py | 130 +------------------------------ 1 file changed, 3 insertions(+), 127 deletions(-) (limited to 'examples') diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py index 864da20e..a40733fb 100644 --- a/examples/complex/block_dns_over_https.py +++ b/examples/complex/block_dns_over_https.py @@ -4,23 +4,12 @@ This module is for blocking DNS over HTTPS requests. It loads a blocklist of IPs and hostnames that are known to serve DNS over HTTPS requests. It also uses headers, query params, and paths to detect DoH (and block it) """ -import json -import re -import os -import urllib.request from typing import List -import dns.query -import dns.rdatatype -import dns.message -import dns.resolver -import dns.rdtypes.IN.A -import dns.rdtypes.IN.AAAA - from mitmproxy import ctx -# filename we'll save the blocklist to so we don't have to re-generate it every time -blocklist_filename = 'blocklist.json' +# known DoH providers' hostnames and IP addresses to block +default_blocklist: dict = {"hostnames": ["dns.adguard.com", "dns-family.adguard.com", "dns.google", "cloudflare-dns.com", "mozilla.cloudflare-dns.com", "security.cloudflare-dns.com", "family.cloudflare-dns.com", "dns.quad9.net", "dns9.quad9.net", "dns10.quad9.net", "dns11.quad9.net", "doh.opendns.com", "doh.familyshield.opendns.com", "doh.cleanbrowsing.org", "doh.xfinity.com", "dohdot.coxlab.net", "odvr.nic.cz", "doh.dnslify.com", "dns.nextdns.io", "dns.dnsoverhttps.net", "doh.crypto.sx", "doh.powerdns.org", "doh-fi.blahdns.com", "doh-jp.blahdns.com", "doh-de.blahdns.com", "doh.ffmuc.net", "dns.dns-over-https.com", "doh.securedns.eu", "dns.rubyfish.cn", "dns.containerpi.com", "dns.containerpi.com", "dns.containerpi.com", "doh-2.seby.io", "doh.seby.io", "commons.host", "doh.dnswarden.com", "doh.dnswarden.com", "doh.dnswarden.com", "dns-nyc.aaflalo.me", "dns.aaflalo.me", "doh.applied-privacy.net", "doh.captnemo.in", "doh.tiar.app", "doh.tiarap.org", "doh.dns.sb", "rdns.faelix.net", "doh.li", "doh.armadillodns.net", "jp.tiar.app", "jp.tiarap.org", "doh.42l.fr", "dns.hostux.net", "dns.hostux.net", "dns.aa.net.uk", "adblock.mydns.network", "ibksturm.synology.me", "jcdns.fun", "ibuki.cgnat.net", "dns.twnic.tw", "example.doh.blockerdns.com", "dns.digitale-gesellschaft.ch", "doh.libredns.gr", "doh.centraleu.pi-dns.com", "doh.northeu.pi-dns.com", "doh.westus.pi-dns.com", "doh.eastus.pi-dns.com", "dns.flatuslifir.is", "private.canadianshield.cira.ca", "protected.canadianshield.cira.ca", "family.canadianshield.cira.ca", "dns.google.com", "dns.google.com"], "ips": ["176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", "2620:10a:80bc::30", "2620:10a:80bb::30"]} # additional hostnames to block additional_doh_names: List[str] = [ @@ -32,120 +21,7 @@ additional_doh_ips: List[str] = [ ] - -def get_doh_providers(): - """ - Scrape a list of DoH providers from curl's wiki page. - :return: a generator of dicts containing information about the DoH providers - """ - https_url_re = re.compile(r'https://' - r'(?P[0-9a-zA-Z._~-]+)' - r'(?P:[0-9]+)?' - r'(?P[0-9a-zA-Z._~/-]+)?') - - provider_re = re.compile(r'(\[([^\]]+)\]\(([^)]+))\)|(.*)') - # URLs that are not DoH URLs - do_not_include = ['my.nextdns.io', 'blog.cloudflare.com'] - found_table = False - with urllib.request.urlopen('https://raw.githubusercontent.com/wiki/curl/curl/DNS-over-HTTPS.md') as fp: - for line in fp: - line = line.decode() - if line.startswith('|'): - if not found_table: - found_table = True - continue - cols = line.split('|') - provider_col = cols[1].strip() - website = None - provider_name = None - matches = provider_re.findall(provider_col) - if matches[0][3] != '': - provider_name = matches[0][3] - if matches[0][1] != '': - provider_name = matches[0][1] - if matches[0][2] != '': - website = matches[0][2] - if provider_name is not None: - provider_name = re.sub(r'([^[]+)\s?(.*)', r'\1', provider_name) - while provider_name[-1] == ' ': - provider_name = provider_name[:-1] - url_col = cols[2] - doh_url_matches = https_url_re.findall(url_col) - if len(doh_url_matches) == 0: - continue - else: - for doh_url in doh_url_matches: - if doh_url[0] in do_not_include: - continue - yield { - 'name': provider_name, - 'website': website, - 'url': 'https://{}{}{}'.format(doh_url[0], - ':{}'.format(doh_url[1]) - if len(doh_url[1]) != 0 - else '', doh_url[2]), - 'hostname': doh_url[0], - 'port': doh_url[1] if len(doh_url[1]) != 0 else '443', - 'path': doh_url[2], - } - if found_table and line.startswith('#'): - break - return - - -def get_ips(hostname): - """ - Lookup all A and AAAA records for given hostname - :param hostname: the name to lookup - :return: a list of IP addresses returned - """ - default_nameserver = dns.resolver.Resolver().nameservers[0] - ips = list() - rdtypes = [dns.rdatatype.A, dns.rdatatype.AAAA] - for rdtype in rdtypes: - q = dns.message.make_query(hostname, rdtype) - r = dns.query.udp(q, default_nameserver) - if r.flags & dns.flags.TC: - r = dns.query.tcp(q, default_nameserver) - for a in r.answer: - for i in a.items: - if isinstance(i, dns.rdtypes.IN.A.A) or isinstance(i, dns.rdtypes.IN.AAAA.AAAA): - ips.append(str(i.address)) - return ips - - -def load_blocklist(): - """ - Load a tuple containing two lists, in the form of (hostnames, ips). - It will attempt to load it from a file, and if that file is not found, - it will generate the blocklist and save it to a file. - - :return: a ``tuple`` of (``list``, ``list``), the hostnames and IPs to block - """ - if os.path.isfile(blocklist_filename): - with open(blocklist_filename, 'r') as fp: - j = json.load(fp) - doh_hostnames, doh_ips = j['hostnames'], j['ips'] - else: - doh_hostnames = list([i['hostname'] for i in get_doh_providers()]) - doh_ips = list() - for hostname in doh_hostnames: - ips = get_ips(hostname) - doh_ips.extend(ips) - doh_hostnames.extend(additional_doh_names) - doh_ips.extend(additional_doh_ips) - with open(blocklist_filename, 'w') as fp: - obj = { - 'hostnames': doh_hostnames, - 'ips': doh_ips - } - json.dump(obj, fp=fp) - return doh_hostnames, doh_ips - - -# load DoH hostnames and IP addresses to block -doh_hostnames, doh_ips = load_blocklist() -ctx.log.info('DoH blocklist loaded') +doh_hostnames, doh_ips = default_blocklist['hostnames'], default_blocklist['ips'] # convert to sets for faster lookups doh_hostnames = set(doh_hostnames) -- cgit v1.2.3 From 83987f9b69d1a19d085c61c7d3c906f0fc9492d4 Mon Sep 17 00:00:00 2001 From: kimbo Date: Sat, 4 Apr 2020 21:27:58 -0600 Subject: broke up long line into multiple lines --- examples/complex/block_dns_over_https.py | 55 +++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) (limited to 'examples') diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py index a40733fb..479f0baa 100644 --- a/examples/complex/block_dns_over_https.py +++ b/examples/complex/block_dns_over_https.py @@ -9,7 +9,60 @@ from typing import List from mitmproxy import ctx # known DoH providers' hostnames and IP addresses to block -default_blocklist: dict = {"hostnames": ["dns.adguard.com", "dns-family.adguard.com", "dns.google", "cloudflare-dns.com", "mozilla.cloudflare-dns.com", "security.cloudflare-dns.com", "family.cloudflare-dns.com", "dns.quad9.net", "dns9.quad9.net", "dns10.quad9.net", "dns11.quad9.net", "doh.opendns.com", "doh.familyshield.opendns.com", "doh.cleanbrowsing.org", "doh.xfinity.com", "dohdot.coxlab.net", "odvr.nic.cz", "doh.dnslify.com", "dns.nextdns.io", "dns.dnsoverhttps.net", "doh.crypto.sx", "doh.powerdns.org", "doh-fi.blahdns.com", "doh-jp.blahdns.com", "doh-de.blahdns.com", "doh.ffmuc.net", "dns.dns-over-https.com", "doh.securedns.eu", "dns.rubyfish.cn", "dns.containerpi.com", "dns.containerpi.com", "dns.containerpi.com", "doh-2.seby.io", "doh.seby.io", "commons.host", "doh.dnswarden.com", "doh.dnswarden.com", "doh.dnswarden.com", "dns-nyc.aaflalo.me", "dns.aaflalo.me", "doh.applied-privacy.net", "doh.captnemo.in", "doh.tiar.app", "doh.tiarap.org", "doh.dns.sb", "rdns.faelix.net", "doh.li", "doh.armadillodns.net", "jp.tiar.app", "jp.tiarap.org", "doh.42l.fr", "dns.hostux.net", "dns.hostux.net", "dns.aa.net.uk", "adblock.mydns.network", "ibksturm.synology.me", "jcdns.fun", "ibuki.cgnat.net", "dns.twnic.tw", "example.doh.blockerdns.com", "dns.digitale-gesellschaft.ch", "doh.libredns.gr", "doh.centraleu.pi-dns.com", "doh.northeu.pi-dns.com", "doh.westus.pi-dns.com", "doh.eastus.pi-dns.com", "dns.flatuslifir.is", "private.canadianshield.cira.ca", "protected.canadianshield.cira.ca", "family.canadianshield.cira.ca", "dns.google.com", "dns.google.com"], "ips": ["176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", "2620:10a:80bc::30", "2620:10a:80bb::30"]} +default_blocklist: dict = { + "hostnames": [ + "dns.adguard.com", "dns-family.adguard.com", "dns.google", "cloudflare-dns.com", + "mozilla.cloudflare-dns.com", "security.cloudflare-dns.com", "family.cloudflare-dns.com", + "dns.quad9.net", "dns9.quad9.net", "dns10.quad9.net", "dns11.quad9.net", "doh.opendns.com", + "doh.familyshield.opendns.com", "doh.cleanbrowsing.org", "doh.xfinity.com", "dohdot.coxlab.net", + "odvr.nic.cz", "doh.dnslify.com", "dns.nextdns.io", "dns.dnsoverhttps.net", "doh.crypto.sx", + "doh.powerdns.org", "doh-fi.blahdns.com", "doh-jp.blahdns.com", "doh-de.blahdns.com", + "doh.ffmuc.net", "dns.dns-over-https.com", "doh.securedns.eu", "dns.rubyfish.cn", + "dns.containerpi.com", "dns.containerpi.com", "dns.containerpi.com", "doh-2.seby.io", + "doh.seby.io", "commons.host", "doh.dnswarden.com", "doh.dnswarden.com", "doh.dnswarden.com", + "dns-nyc.aaflalo.me", "dns.aaflalo.me", "doh.applied-privacy.net", "doh.captnemo.in", + "doh.tiar.app", "doh.tiarap.org", "doh.dns.sb", "rdns.faelix.net", "doh.li", "doh.armadillodns.net", + "jp.tiar.app", "jp.tiarap.org", "doh.42l.fr", "dns.hostux.net", "dns.hostux.net", "dns.aa.net.uk", + "adblock.mydns.network", "ibksturm.synology.me", "jcdns.fun", "ibuki.cgnat.net", "dns.twnic.tw", + "example.doh.blockerdns.com", "dns.digitale-gesellschaft.ch", "doh.libredns.gr", + "doh.centraleu.pi-dns.com", "doh.northeu.pi-dns.com", "doh.westus.pi-dns.com", + "doh.eastus.pi-dns.com", "dns.flatuslifir.is", "private.canadianshield.cira.ca", + "protected.canadianshield.cira.ca", "family.canadianshield.cira.ca", "dns.google.com", + "dns.google.com" + ], + "ips": [ + "176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", + "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", + "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", + "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", + "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", + "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", + "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", + "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", + "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", + "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", + "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", + "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", + "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", + "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", + "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", + "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", + "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", + "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", + "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", + "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", + "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", + "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", + "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", + "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", + "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", + "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", + "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", + "2620:10a:80bc::30", "2620:10a:80bb::30" + ] +} # additional hostnames to block additional_doh_names: List[str] = [ -- cgit v1.2.3