diff mbox series

[2/2] importer.py: add source information for RIR data feeds

Message ID 20210515115705.9794-2-peter.mueller@ipfire.org
State Superseded
Headers
Series [1/2] location-importer.in: keep track of sources for networks, ASNs, and organisations |

Commit Message

Peter Müller May 15, 2021, 11:57 a.m. UTC 
  Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
---
 src/python/importer.py | 66 +++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 24 deletions(-)
diff mbox series

Patch

diff --git a/src/python/importer.py b/src/python/importer.py
index 5f46bc3..4c8406c 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -25,50 +25,68 @@  import urllib.request
 log = logging.getLogger("location.importer")
 log.propagate = 1
 
-WHOIS_SOURCES = (
+WHOIS_SOURCES = {
 	# African Network Information Centre
-	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
+	"AFRINIC": [
+		"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz"
+		],
 
 	# Asia Pacific Network Information Centre
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
-	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
-	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
-	"https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz",
+	"APNIC": [
+		"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+		"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+		#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
+		#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
+		"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
+		"https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz"
+		],
 
 	# American Registry for Internet Numbers
 	# XXX there is nothing useful for us in here
-	#"https://ftp.arin.net/pub/rr/arin.db",
+	# ARIN: [
+	#	"https://ftp.arin.net/pub/rr/arin.db"
+	# ],
 
 	# Latin America and Caribbean Network Information Centre
 	# XXX ???
 
 	# Réseaux IP Européens
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
-	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz",
-)
-
-EXTENDED_SOURCES = (
+	"RIPE": [
+		"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+		"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+		#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
+		#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
+		"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
+		"https://ftp.ripe.net/ripe/dbase/split/ripe.db.organisation.gz"
+		],
+}
+
+EXTENDED_SOURCES = {
 	# African Network Information Centre
-	#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+	# "ARIN": [
+	#	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest"
+	# ],
 
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+	# "APNIC": [
+	#	"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest"
+	# ],
 
 	# American Registry for Internet Numbers
-	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
+	"ARIN": [
+		"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest"
+		],
 
 	# Latin America and Caribbean Network Information Centre
-	"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
+	"LACNIC": [
+		"https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest"
+		],
 
 	# Réseaux IP Européens
-	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
-)
+	# "RIPE": [
+	#	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest"
+	# ],
+}
 
 class Downloader(object):
 	def __init__(self):