From f36a5b8aa8aaa3576a1ee962516120a9de0cd03c Mon Sep 17 00:00:00 2001 From: kimbo Date: Sat, 4 Apr 2020 21:18:58 -0600 Subject: replace scraping and DNS lookups with static list --- examples/complex/block_dns_over_https.py | 130 +------------------------------ 1 file changed, 3 insertions(+), 127 deletions(-) (limited to 'examples') diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py index 864da20e..a40733fb 100644 --- a/examples/complex/block_dns_over_https.py +++ b/examples/complex/block_dns_over_https.py @@ -4,23 +4,12 @@ This module is for blocking DNS over HTTPS requests. It loads a blocklist of IPs and hostnames that are known to serve DNS over HTTPS requests. It also uses headers, query params, and paths to detect DoH (and block it) """ -import json -import re -import os -import urllib.request from typing import List -import dns.query -import dns.rdatatype -import dns.message -import dns.resolver -import dns.rdtypes.IN.A -import dns.rdtypes.IN.AAAA - from mitmproxy import ctx -# filename we'll save the blocklist to so we don't have to re-generate it every time -blocklist_filename = 'blocklist.json' +# known DoH providers' hostnames and IP addresses to block +default_blocklist: dict = {"hostnames": ["dns.adguard.com", "dns-family.adguard.com", "dns.google", "cloudflare-dns.com", "mozilla.cloudflare-dns.com", "security.cloudflare-dns.com", "family.cloudflare-dns.com", "dns.quad9.net", "dns9.quad9.net", "dns10.quad9.net", "dns11.quad9.net", "doh.opendns.com", "doh.familyshield.opendns.com", "doh.cleanbrowsing.org", "doh.xfinity.com", "dohdot.coxlab.net", "odvr.nic.cz", "doh.dnslify.com", "dns.nextdns.io", "dns.dnsoverhttps.net", "doh.crypto.sx", "doh.powerdns.org", "doh-fi.blahdns.com", "doh-jp.blahdns.com", "doh-de.blahdns.com", "doh.ffmuc.net", "dns.dns-over-https.com", "doh.securedns.eu", "dns.rubyfish.cn", "dns.containerpi.com", "dns.containerpi.com", "dns.containerpi.com", "doh-2.seby.io", "doh.seby.io", "commons.host", "doh.dnswarden.com", "doh.dnswarden.com", "doh.dnswarden.com", "dns-nyc.aaflalo.me", "dns.aaflalo.me", "doh.applied-privacy.net", "doh.captnemo.in", "doh.tiar.app", "doh.tiarap.org", "doh.dns.sb", "rdns.faelix.net", "doh.li", "doh.armadillodns.net", "jp.tiar.app", "jp.tiarap.org", "doh.42l.fr", "dns.hostux.net", "dns.hostux.net", "dns.aa.net.uk", "adblock.mydns.network", "ibksturm.synology.me", "jcdns.fun", "ibuki.cgnat.net", "dns.twnic.tw", "example.doh.blockerdns.com", "dns.digitale-gesellschaft.ch", "doh.libredns.gr", "doh.centraleu.pi-dns.com", "doh.northeu.pi-dns.com", "doh.westus.pi-dns.com", "doh.eastus.pi-dns.com", "dns.flatuslifir.is", "private.canadianshield.cira.ca", "protected.canadianshield.cira.ca", "family.canadianshield.cira.ca", "dns.google.com", "dns.google.com"], "ips": ["176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", "2620:10a:80bc::30", "2620:10a:80bb::30"]} # additional hostnames to block additional_doh_names: List[str] = [ @@ -32,120 +21,7 @@ additional_doh_ips: List[str] = [ ] - -def get_doh_providers(): - """ - Scrape a list of DoH providers from curl's wiki page. - :return: a generator of dicts containing information about the DoH providers - """ - https_url_re = re.compile(r'https://' - r'(?P[0-9a-zA-Z._~-]+)' - r'(?P:[0-9]+)?' - r'(?P[0-9a-zA-Z._~/-]+)?') - - provider_re = re.compile(r'(\[([^\]]+)\]\(([^)]+))\)|(.*)') - # URLs that are not DoH URLs - do_not_include = ['my.nextdns.io', 'blog.cloudflare.com'] - found_table = False - with urllib.request.urlopen('https://raw.githubusercontent.com/wiki/curl/curl/DNS-over-HTTPS.md') as fp: - for line in fp: - line = line.decode() - if line.startswith('|'): - if not found_table: - found_table = True - continue - cols = line.split('|') - provider_col = cols[1].strip() - website = None - provider_name = None - matches = provider_re.findall(provider_col) - if matches[0][3] != '': - provider_name = matches[0][3] - if matches[0][1] != '': - provider_name = matches[0][1] - if matches[0][2] != '': - website = matches[0][2] - if provider_name is not None: - provider_name = re.sub(r'([^[]+)\s?(.*)', r'\1', provider_name) - while provider_name[-1] == ' ': - provider_name = provider_name[:-1] - url_col = cols[2] - doh_url_matches = https_url_re.findall(url_col) - if len(doh_url_matches) == 0: - continue - else: - for doh_url in doh_url_matches: - if doh_url[0] in do_not_include: - continue - yield { - 'name': provider_name, - 'website': website, - 'url': 'https://{}{}{}'.format(doh_url[0], - ':{}'.format(doh_url[1]) - if len(doh_url[1]) != 0 - else '', doh_url[2]), - 'hostname': doh_url[0], - 'port': doh_url[1] if len(doh_url[1]) != 0 else '443', - 'path': doh_url[2], - } - if found_table and line.startswith('#'): - break - return - - -def get_ips(hostname): - """ - Lookup all A and AAAA records for given hostname - :param hostname: the name to lookup - :return: a list of IP addresses returned - """ - default_nameserver = dns.resolver.Resolver().nameservers[0] - ips = list() - rdtypes = [dns.rdatatype.A, dns.rdatatype.AAAA] - for rdtype in rdtypes: - q = dns.message.make_query(hostname, rdtype) - r = dns.query.udp(q, default_nameserver) - if r.flags & dns.flags.TC: - r = dns.query.tcp(q, default_nameserver) - for a in r.answer: - for i in a.items: - if isinstance(i, dns.rdtypes.IN.A.A) or isinstance(i, dns.rdtypes.IN.AAAA.AAAA): - ips.append(str(i.address)) - return ips - - -def load_blocklist(): - """ - Load a tuple containing two lists, in the form of (hostnames, ips). - It will attempt to load it from a file, and if that file is not found, - it will generate the blocklist and save it to a file. - - :return: a ``tuple`` of (``list``, ``list``), the hostnames and IPs to block - """ - if os.path.isfile(blocklist_filename): - with open(blocklist_filename, 'r') as fp: - j = json.load(fp) - doh_hostnames, doh_ips = j['hostnames'], j['ips'] - else: - doh_hostnames = list([i['hostname'] for i in get_doh_providers()]) - doh_ips = list() - for hostname in doh_hostnames: - ips = get_ips(hostname) - doh_ips.extend(ips) - doh_hostnames.extend(additional_doh_names) - doh_ips.extend(additional_doh_ips) - with open(blocklist_filename, 'w') as fp: - obj = { - 'hostnames': doh_hostnames, - 'ips': doh_ips - } - json.dump(obj, fp=fp) - return doh_hostnames, doh_ips - - -# load DoH hostnames and IP addresses to block -doh_hostnames, doh_ips = load_blocklist() -ctx.log.info('DoH blocklist loaded') +doh_hostnames, doh_ips = default_blocklist['hostnames'], default_blocklist['ips'] # convert to sets for faster lookups doh_hostnames = set(doh_hostnames) -- cgit v1.2.3