replace scraping and DNS lookups with static list

1 files changed, 3 insertions, 127 deletions

diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py
index 864da20e..a40733fb 100644
--- a/examples/complex/block_dns_over_https.py
+++ b/examples/complex/block_dns_over_https.py
@@ -4,23 +4,12 @@ This module is for blocking DNS over HTTPS requests.
 It loads a blocklist of IPs and hostnames that are known to serve DNS over HTTPS requests.
 It also uses headers, query params, and paths to detect DoH (and block it)
 """
-import json
-import re
-import os
-import urllib.request
 from typing import List
 
-import dns.query
-import dns.rdatatype
-import dns.message
-import dns.resolver
-import dns.rdtypes.IN.A
-import dns.rdtypes.IN.AAAA
-
 from mitmproxy import ctx
 
-# filename we'll save the blocklist to so we don't have to re-generate it every time
-blocklist_filename = 'blocklist.json'
+# known DoH providers' hostnames and IP addresses to block
+default_blocklist: dict = {"hostnames": ["dns.adguard.com", "dns-family.adguard.com", "dns.google", "cloudflare-dns.com", "mozilla.cloudflare-dns.com", "security.cloudflare-dns.com", "family.cloudflare-dns.com", "dns.quad9.net", "dns9.quad9.net", "dns10.quad9.net", "dns11.quad9.net", "doh.opendns.com", "doh.familyshield.opendns.com", "doh.cleanbrowsing.org", "doh.xfinity.com", "dohdot.coxlab.net", "odvr.nic.cz", "doh.dnslify.com", "dns.nextdns.io", "dns.dnsoverhttps.net", "doh.crypto.sx", "doh.powerdns.org", "doh-fi.blahdns.com", "doh-jp.blahdns.com", "doh-de.blahdns.com", "doh.ffmuc.net", "dns.dns-over-https.com", "doh.securedns.eu", "dns.rubyfish.cn", "dns.containerpi.com", "dns.containerpi.com", "dns.containerpi.com", "doh-2.seby.io", "doh.seby.io", "commons.host", "doh.dnswarden.com", "doh.dnswarden.com", "doh.dnswarden.com", "dns-nyc.aaflalo.me", "dns.aaflalo.me", "doh.applied-privacy.net", "doh.captnemo.in", "doh.tiar.app", "doh.tiarap.org", "doh.dns.sb", "rdns.faelix.net", "doh.li", "doh.armadillodns.net", "jp.tiar.app", "jp.tiarap.org", "doh.42l.fr", "dns.hostux.net", "dns.hostux.net", "dns.aa.net.uk", "adblock.mydns.network", "ibksturm.synology.me", "jcdns.fun", "ibuki.cgnat.net", "dns.twnic.tw", "example.doh.blockerdns.com", "dns.digitale-gesellschaft.ch", "doh.libredns.gr", "doh.centraleu.pi-dns.com", "doh.northeu.pi-dns.com", "doh.westus.pi-dns.com", "doh.eastus.pi-dns.com", "dns.flatuslifir.is", "private.canadianshield.cira.ca", "protected.canadianshield.cira.ca", "family.canadianshield.cira.ca", "dns.google.com", "dns.google.com"], "ips": ["176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", "2620:10a:80bc::30", "2620:10a:80bb::30"]}
 
 # additional hostnames to block
 additional_doh_names: List[str] = [
@@ -32,120 +21,7 @@ additional_doh_ips: List[str] = [
 
 ]
 
-
-def get_doh_providers():
-    """
-    Scrape a list of DoH providers from curl's wiki page.
-    :return: a generator of dicts containing information about the DoH providers
-    """
-    https_url_re = re.compile(r'https://'
-                              r'(?P<hostname>[0-9a-zA-Z._~-]+)'
-                              r'(?P<port>:[0-9]+)?'
-                              r'(?P<path>[0-9a-zA-Z._~/-]+)?')
-
-    provider_re = re.compile(r'(\[([^\]]+)\]\(([^)]+))\)|(.*)')
-    # URLs that are not DoH URLs
-    do_not_include = ['my.nextdns.io', 'blog.cloudflare.com']
-    found_table = False
-    with urllib.request.urlopen('https://raw.githubusercontent.com/wiki/curl/curl/DNS-over-HTTPS.md') as fp:
-        for line in fp:
-            line = line.decode()
-            if line.startswith('|'):
-                if not found_table:
-                    found_table = True
-                    continue
-                cols = line.split('|')
-                provider_col = cols[1].strip()
-                website = None
-                provider_name = None
-                matches = provider_re.findall(provider_col)
-                if matches[0][3] != '':
-                    provider_name = matches[0][3]
-                if matches[0][1] != '':
-                    provider_name = matches[0][1]
-                if matches[0][2] != '':
-                    website = matches[0][2]
-                if provider_name is not None:
-                    provider_name = re.sub(r'([^[]+)\s?(.*)', r'\1', provider_name)
-                    while provider_name[-1] == ' ':
-                        provider_name = provider_name[:-1]
-                url_col = cols[2]
-                doh_url_matches = https_url_re.findall(url_col)
-                if len(doh_url_matches) == 0:
-                    continue
-                else:
-                    for doh_url in doh_url_matches:
-                        if doh_url[0] in do_not_include:
-                            continue
-                        yield {
-                            'name': provider_name,
-                            'website': website,
-                            'url': 'https://{}{}{}'.format(doh_url[0],
-                                                           ':{}'.format(doh_url[1])
-                                                           if len(doh_url[1]) != 0
-                                                           else '', doh_url[2]),
-                            'hostname': doh_url[0],
-                            'port': doh_url[1] if len(doh_url[1]) != 0 else '443',
-                            'path': doh_url[2],
-                        }
-            if found_table and line.startswith('#'):
-                break
-    return
-
-
-def get_ips(hostname):
-    """
-    Lookup all A and AAAA records for given hostname
-    :param hostname: the name to lookup
-    :return: a list of IP addresses returned
-    """
-    default_nameserver = dns.resolver.Resolver().nameservers[0]
-    ips = list()
-    rdtypes = [dns.rdatatype.A, dns.rdatatype.AAAA]
-    for rdtype in rdtypes:
-        q = dns.message.make_query(hostname, rdtype)
-        r = dns.query.udp(q, default_nameserver)
-        if r.flags & dns.flags.TC:
-            r = dns.query.tcp(q, default_nameserver)
-        for a in r.answer:
-            for i in a.items:
-                if isinstance(i, dns.rdtypes.IN.A.A) or isinstance(i, dns.rdtypes.IN.AAAA.AAAA):
-                    ips.append(str(i.address))
-    return ips
-
-
-def load_blocklist():
-    """
-    Load a tuple containing two lists, in the form of (hostnames, ips).
-    It will attempt to load it from a file, and if that file is not found,
-    it will generate the blocklist and save it to a file.
-
-    :return: a ``tuple`` of (``list``, ``list``), the hostnames and IPs to block
-    """
-    if os.path.isfile(blocklist_filename):
-        with open(blocklist_filename, 'r') as fp:
-            j = json.load(fp)
-        doh_hostnames, doh_ips = j['hostnames'], j['ips']
-    else:
-        doh_hostnames = list([i['hostname'] for i in get_doh_providers()])
-        doh_ips = list()
-        for hostname in doh_hostnames:
-            ips = get_ips(hostname)
-            doh_ips.extend(ips)
-    doh_hostnames.extend(additional_doh_names)
-    doh_ips.extend(additional_doh_ips)
-    with open(blocklist_filename, 'w') as fp:
-        obj = {
-            'hostnames': doh_hostnames,
-            'ips': doh_ips
-        }
-        json.dump(obj, fp=fp)
-    return doh_hostnames, doh_ips
-
-
-# load DoH hostnames and IP addresses to block
-doh_hostnames, doh_ips = load_blocklist()
-ctx.log.info('DoH blocklist loaded')
+doh_hostnames, doh_ips = default_blocklist['hostnames'], default_blocklist['ips']
 
 # convert to sets for faster lookups
 doh_hostnames = set(doh_hostnames)