From patchwork Wed Oct 21 14:47:41 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Peter_M=C3=BCller?= X-Patchwork-Id: 3597 Return-Path: Received: from mail01.ipfire.org (mail01.haj.ipfire.org [172.28.1.202]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-384) client-signature ECDSA (P-384)) (Client CN "mail01.haj.ipfire.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by web04.haj.ipfire.org (Postfix) with ESMTPS id 4CGYJB3bS4z3wgp for ; Wed, 21 Oct 2020 14:47:54 +0000 (UTC) Received: from mail02.haj.ipfire.org (mail02.haj.ipfire.org [172.28.1.201]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-384) client-signature ECDSA (P-384)) (Client CN "mail02.haj.ipfire.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mail01.ipfire.org (Postfix) with ESMTPS id 4CGYJ96zntz18q; Wed, 21 Oct 2020 14:47:53 +0000 (UTC) Received: from mail02.haj.ipfire.org (localhost [127.0.0.1]) by mail02.haj.ipfire.org (Postfix) with ESMTP id 4CGYJ96f6dz2yPl; Wed, 21 Oct 2020 14:47:53 +0000 (UTC) Received: from mail01.ipfire.org (mail01.haj.ipfire.org [172.28.1.202]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-384) client-signature ECDSA (P-384)) (Client CN "mail01.haj.ipfire.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mail02.haj.ipfire.org (Postfix) with ESMTPS id 4CGYJ769TXz2xyS for ; Wed, 21 Oct 2020 14:47:51 +0000 (UTC) Received: from location02.haj.ipfire.org (location02.haj.ipfire.org [172.28.1.170]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-384) client-signature ECDSA (P-384)) (Client CN "location02.haj.ipfire.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mail01.ipfire.org (Postfix) with ESMTPS id 4CGYJ74xrwz11M; Wed, 21 Oct 2020 14:47:51 +0000 (UTC) Received: by location02.haj.ipfire.org (Postfix, from userid 0) id 4CGYJ70xg2z13cx; Wed, 21 Oct 2020 14:47:51 +0000 (UTC) From: =?utf-8?q?Peter_M=C3=BCller?= To: location@lists.ipfire.org Subject: [PATCH 6/8] location-importer.in: omit historic/orphaned RIR data Date: Wed, 21 Oct 2020 14:47:41 +0000 Message-Id: <20201021144743.18083-6-peter.mueller@ipfire.org> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201021144743.18083-1-peter.mueller@ipfire.org> References: <20201021144743.18083-1-peter.mueller@ipfire.org> MIME-Version: 1.0 X-BeenThere: location@lists.ipfire.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: location-bounces@lists.ipfire.org Sender: "Location" Some RIRs include detailled information regarding networks not managed by or allocated to themselves, particually APNIC. We need to filter those networks (they usually have a characteristic network name) in order to prevent operational quirks or returning wrong country codes. Fixes: #12501 Partially fixes: #12499 Cc: Michael Tremer Signed-off-by: Peter Müller --- src/python/location-importer.in | 38 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/python/location-importer.in b/src/python/location-importer.in index 20eb052..4f4a46d 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -484,38 +484,38 @@ class CLI(object): return False if not network.is_global: - logging.warning("Skipping non-globally routable network: %s" % network) + log.warning("Skipping non-globally routable network: %s" % network) return False if network.version == 4: if network.prefixlen < 7: - logging.warning("Skipping too big IP chunk: %s" % network) + log.warning("Skipping too big IP chunk: %s" % network) return False if network.prefixlen > 24: - logging.info("Skipping network too small to be publicly announced: %s" % network) + log.info("Skipping network too small to be publicly announced: %s" % network) return False if str(network.network_address) == "0.0.0.0": - logging.warning("Skipping network based on 0.0.0.0: %s" % network) + log.warning("Skipping network based on 0.0.0.0: %s" % network) return False elif network.version == 6: if network.prefixlen < 10: - logging.warning("Skipping too big IP chunk: %s" % network) + log.warning("Skipping too big IP chunk: %s" % network) return False if network.prefixlen > 48: - logging.info("Skipping network too small to be publicly announced: %s" % network) + log.info("Skipping network too small to be publicly announced: %s" % network) return False if str(network.network_address) == "::": - logging.warning("Skipping network based on '::': %s" % network) + log.warning("Skipping network based on '::': %s" % network) return False else: # This should not happen... - logging.warning("Skipping network of unknown family, this should not happen: %s" % network) + log.warning("Skipping network of unknown family, this should not happen: %s" % network) return False # In case we have made it here, the network is considered to @@ -564,15 +564,22 @@ class CLI(object): ) def _parse_inetnum_block(self, block): - logging.debug("Parsing inetnum block:") + log.debug("Parsing inetnum block:") inetnum = {} for line in block: - logging.debug(line) + log.debug(line) # Split line key, val = split_line(line) + # Filter any inetnum records which are only referring to IP space + # not managed by that specific RIR... + if key == "netname": + if re.match(r"(ERX-NETBLOCK|(AFRINIC|ARIN|LACNIC|RIPE)-CIDR-BLOCK|IANA-NETBLOCK-\d{1,3}|NON-RIPE-NCC-MANAGED-ADDRESS-BLOCK)", val.strip()): + log.warning("Skipping record indicating historic/orphaned data: %s" % val.strip()) + return + if key == "inetnum": start_address, delim, end_address = val.partition("-") @@ -584,7 +591,7 @@ class CLI(object): start_address = ipaddress.ip_address(start_address) end_address = ipaddress.ip_address(end_address) except ValueError: - logging.warning("Could not parse line: %s" % line) + log.warning("Could not parse line: %s" % line) return # Set prefix to default @@ -601,15 +608,18 @@ class CLI(object): inetnum[key] = val elif key == "country": - if val == "UNITED STATES": - val = "US" - inetnum[key] = val.upper() # Skip empty objects if not inetnum or not "country" in inetnum: return + # Skip objects with bogus country code 'ZZ' + if inetnum.get("country") == "ZZ": + log.warning("Skipping network with bogus country 'ZZ': %s" % \ + (inetnum.get("inet6num") or inetnum.get("inetnum"))) + return + network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False) if not self._check_parsed_network(network):