@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
# Asia Pacific Network Information Centre
- #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
- #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+ "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+ "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
# XXX ???
# Réseaux IP Européens
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
- #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+ "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
EXTENDED_SOURCES = (
# African Network Information Centre
- "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+ #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
# Asia Pacific Network Information Centre
- "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+ #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
# American Registry for Internet Numbers
"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
# Réseaux IP Européens
- "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+ #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
)
class Downloader(object):
@@ -395,6 +395,10 @@ class CLI(object):
if line.startswith("aut-num:"):
return self._parse_autnum_block(block)
+ # inetnum
+ if line.startswith("inet6num:") or line.startswith("inetnum:"):
+ return self._parse_inetnum_block(block)
+
# organisation
elif line.startswith("organisation:"):
return self._parse_org_block(block)
@@ -424,6 +428,65 @@ class CLI(object):
autnum.get("asn"), autnum.get("org"),
)
+ def _parse_inetnum_block(self, block):
+ logging.debug("Parsing inetnum block:")
+
+ inetnum = {}
+ for line in block:
+ logging.debug(line)
+
+ # Split line
+ key, val = split_line(line)
+
+ if key == "inetnum":
+ start_address, delim, end_address = val.partition("-")
+
+ # Strip any excess space
+ start_address, end_address = start_address.rstrip(), end_address.strip()
+
+ # Convert to IP address
+ try:
+ start_address = ipaddress.ip_address(start_address)
+ end_address = ipaddress.ip_address(end_address)
+ except ValueError:
+ logging.warning("Could not parse line: %s" % line)
+ return
+
+ # Set prefix to default
+ prefix = 32
+
+ # Count number of addresses in this subnet
+ num_addresses = int(end_address) - int(start_address)
+ if num_addresses:
+ prefix -= math.log(num_addresses, 2)
+
+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+
+ elif key == "inet6num":
+ inetnum[key] = val
+
+ elif key == "country":
+ if val == "UNITED STATES":
+ val = "US"
+
+ inetnum[key] = val.upper()
+
+ # Skip empty objects
+ if not inetnum:
+ return
+
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+ # Bail out in case we have processed a non-public IP network
+ if network.is_private:
+ logging.warning("Skipping non-globally routable network: %s" % network)
+ return
+
+ self.db.execute("INSERT INTO networks(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+ "%s" % network, inetnum.get("country"),
+ )
+
def _parse_org_block(self, block):
org = {}
for line in block: