Implement an additional flag for hostile networks safe to drop
Commit Message
This patch implements an additional flag intended for networks and
Autonomous Systems being considered hostile. While libloc does not and
should not be an opinionated database, reality shows it is being used
this way.
Hereby, we assign "XD" (drop) as a custom country code for networks
being flagged this way. According to ISO, "XA" to "XZ" are reserved for
"user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html),
so this is a safe thing to do.
This patch does not interfere with "A1" to "A3", which we currently
assign outside standardised country code ranges for historical reasons.
Neither does it specify any policy or source for tagging networks with a
"drop" flag. Doing so is beyond the scope of this - technical -
approach.
To avoid confusions with the SQL "DROP" command, "is_drop" will be used
as a column name for database operations.
Thanks to Michael for his remarks and ideas during the run-up.
Cc: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
---
man/location.txt | 4 ++--
po/de.po | 3 +++
src/loc/network.h | 3 ++-
src/perl/Location.xs | 2 ++
src/python/export.py | 3 ++-
src/python/location-importer.in | 32 +++++++++++++++++++++++++++-----
src/python/location.in | 19 ++++++++++++++++---
src/python/locationmodule.c | 5 ++++-
8 files changed, 58 insertions(+), 13 deletions(-)
Comments
Hello,
Thank you. Great work. Merged.
-Michael
> On 4 Jun 2021, at 16:57, Peter Müller <peter.mueller@ipfire.org> wrote:
>
> This patch implements an additional flag intended for networks and
> Autonomous Systems being considered hostile. While libloc does not and
> should not be an opinionated database, reality shows it is being used
> this way.
>
> Hereby, we assign "XD" (drop) as a custom country code for networks
> being flagged this way. According to ISO, "XA" to "XZ" are reserved for
> "user-assgined codes" (https://www.iso.org/glossary-for-iso-3166.html),
> so this is a safe thing to do.
>
> This patch does not interfere with "A1" to "A3", which we currently
> assign outside standardised country code ranges for historical reasons.
>
> Neither does it specify any policy or source for tagging networks with a
> "drop" flag. Doing so is beyond the scope of this - technical -
> approach.
>
> To avoid confusions with the SQL "DROP" command, "is_drop" will be used
> as a column name for database operations.
>
> Thanks to Michael for his remarks and ideas during the run-up.
>
> Cc: Michael Tremer <michael.tremer@ipfire.org>
> Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
> ---
> man/location.txt | 4 ++--
> po/de.po | 3 +++
> src/loc/network.h | 3 ++-
> src/perl/Location.xs | 2 ++
> src/python/export.py | 3 ++-
> src/python/location-importer.in | 32 +++++++++++++++++++++++++++-----
> src/python/location.in | 19 ++++++++++++++++---
> src/python/locationmodule.c | 5 ++++-
> 8 files changed, 58 insertions(+), 13 deletions(-)
>
> diff --git a/man/location.txt b/man/location.txt
> index df1da53..b38f21c 100644
> --- a/man/location.txt
> +++ b/man/location.txt
> @@ -10,7 +10,7 @@ location - Query the location database
> `location list-countries [--show-name] [--show-continent]`
> `location list-networks-by-as ASN`
> `location list-networks-by-cc COUNTRY_CODE`
> -`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
> +`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]`
> `location lookup ADDRESS [ADDRESS...]`
> `location search-as STRING`
> `location update [--cron=daily|weekly|monthly]`
> @@ -73,7 +73,7 @@ or countries.
> +
> See above for usage of the '--family' and '--format' parameters.
>
> -'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]'::
> +'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]'::
> Lists all networks that have a certain flag.
> +
> See above for usage of the '--family' and '--format' parameters.
> diff --git a/po/de.po b/po/de.po
> index 3b073d6..3cbcdd7 100644
> --- a/po/de.po
> +++ b/po/de.po
> @@ -152,6 +152,9 @@ msgstr ""
> msgid "Anycasts"
> msgstr ""
>
> +msgid "Hostile Networks safe to drop"
> +msgstr ""
> +
> msgid "Lists all countries"
> msgstr ""
>
> diff --git a/src/loc/network.h b/src/loc/network.h
> index af3dafd..a30f653 100644
> --- a/src/loc/network.h
> +++ b/src/loc/network.h
> @@ -1,7 +1,7 @@
> /*
> libloc - A library to determine the location of someone on the Internet
>
> - Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
> + Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>
>
> This library is free software; you can redistribute it and/or
> modify it under the terms of the GNU Lesser General Public
> @@ -27,6 +27,7 @@ enum loc_network_flags {
> LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
> LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2
> LOC_NETWORK_FLAG_ANYCAST = (1 << 2), // A3
> + LOC_NETWORK_FLAG_DROP = (1 << 3), // XD
> };
>
> struct loc_network;
> diff --git a/src/perl/Location.xs b/src/perl/Location.xs
> index b7676d2..73f85b4 100644
> --- a/src/perl/Location.xs
> +++ b/src/perl/Location.xs
> @@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag)
> iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER;
> else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0)
> iv |= LOC_NETWORK_FLAG_ANYCAST;
> + else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0)
> + iv |= LOC_NETWORK_FLAG_DROP;
> else
> croak("Invalid flag");
>
> diff --git a/src/python/export.py b/src/python/export.py
> index f0eae26..3b9e1e0 100644
> --- a/src/python/export.py
> +++ b/src/python/export.py
> @@ -3,7 +3,7 @@
> # #
> # libloc - A library to determine the location of someone on the Internet #
> # #
> -# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
> +# Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
> # #
> # This library is free software; you can redistribute it and/or #
> # modify it under the terms of the GNU Lesser General Public #
> @@ -33,6 +33,7 @@ FLAGS = {
> _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
> _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
> _location.NETWORK_FLAG_ANYCAST : "A3",
> + _location.NETWORK_FLAG_DROP : "XD",
> }
>
> class OutputWriter(object):
> diff --git a/src/python/location-importer.in b/src/python/location-importer.in
> index f796652..ea1e8f5 100644
> --- a/src/python/location-importer.in
> +++ b/src/python/location-importer.in
> @@ -182,6 +182,7 @@ class CLI(object):
> );
> CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
> ON autnum_overrides(number);
> + ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
>
> CREATE TABLE IF NOT EXISTS network_overrides(
> network inet NOT NULL,
> @@ -194,6 +195,7 @@ class CLI(object):
> ON network_overrides(network);
> CREATE INDEX IF NOT EXISTS network_overrides_search
> ON network_overrides USING GIST(network inet_ops);
> + ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
> """)
>
> return db
> @@ -301,7 +303,20 @@ class CLI(object):
> WHERE networks.autnum = overrides.number
> ),
> FALSE
> - ) AS is_anycast
> + ) AS is_anycast,
> + COALESCE(
> + (
> + SELECT is_drop FROM network_overrides overrides
> + WHERE networks.network <<= overrides.network
> + ORDER BY masklen(overrides.network) DESC
> + LIMIT 1
> + ),
> + (
> + SELECT is_drop FROM autnum_overrides overrides
> + WHERE networks.autnum = overrides.number
> + ),
> + FALSE
> + ) AS is_drop
> FROM (
> SELECT
> known_networks.network AS network,
> @@ -350,6 +365,9 @@ class CLI(object):
> if row.is_anycast:
> network.set_flag(location.NETWORK_FLAG_ANYCAST)
>
> + if row.is_drop:
> + network.set_flag(location.NETWORK_FLAG_DROP)
> +
> # Add all countries
> log.info("Writing countries...")
> rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
> @@ -966,14 +984,16 @@ class CLI(object):
> country,
> is_anonymous_proxy,
> is_satellite_provider,
> - is_anycast
> - ) VALUES (%s, %s, %s, %s, %s)
> + is_anycast,
> + is_drop
> + ) VALUES (%s, %s, %s, %s, %s, %s)
> ON CONFLICT (network) DO NOTHING""",
> "%s" % network,
> block.get("country"),
> self._parse_bool(block, "is-anonymous-proxy"),
> self._parse_bool(block, "is-satellite-provider"),
> self._parse_bool(block, "is-anycast"),
> + self._parse_bool(block, "drop"),
> )
>
> elif type == "aut-num":
> @@ -994,8 +1014,9 @@ class CLI(object):
> country,
> is_anonymous_proxy,
> is_satellite_provider,
> - is_anycast
> - ) VALUES(%s, %s, %s, %s, %s, %s)
> + is_anycast,
> + is_drop
> + ) VALUES(%s, %s, %s, %s, %s, %s, %s)
> ON CONFLICT DO NOTHING""",
> autnum,
> block.get("name"),
> @@ -1003,6 +1024,7 @@ class CLI(object):
> self._parse_bool(block, "is-anonymous-proxy"),
> self._parse_bool(block, "is-satellite-provider"),
> self._parse_bool(block, "is-anycast"),
> + self._parse_bool(block, "drop"),
> )
>
> else:
> diff --git a/src/python/location.in b/src/python/location.in
> index e02b4e8..0c89d75 100644
> --- a/src/python/location.in
> +++ b/src/python/location.in
> @@ -3,7 +3,7 @@
> # #
> # libloc - A library to determine the location of someone on the Internet #
> # #
> -# Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
> +# Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org> #
> # #
> # This library is free software; you can redistribute it and/or #
> # modify it under the terms of the GNU Lesser General Public #
> @@ -146,6 +146,9 @@ class CLI(object):
> list_networks_by_flags.add_argument("--anycast",
> action="store_true", help=_("Anycasts"),
> )
> + list_networks_by_flags.add_argument("--drop",
> + action="store_true", help=_("Hostile Networks safe to drop"),
> + )
> list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
> list_networks_by_flags.add_argument("--format",
> choices=location.export.formats.keys(), default="list")
> @@ -305,6 +308,12 @@ class CLI(object):
> _("Anycast"), _("yes"),
> ))
>
> + # Hostile Network
> + if network.has_flag(location.NETWORK_FLAG_DROP):
> + print(format % (
> + _("Hostile Network safe to drop"), _("yes"),
> + ))
> +
> return ret
>
> def handle_dump(self, db, ns):
> @@ -346,6 +355,7 @@ class CLI(object):
> location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
> location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
> location.NETWORK_FLAG_ANYCAST : "is-anycast:",
> + location.NETWORK_FLAG_DROP : "drop:",
> }
>
> # Iterate over all networks
> @@ -523,6 +533,9 @@ class CLI(object):
> if ns.anycast:
> flags |= location.NETWORK_FLAG_ANYCAST
>
> + if ns.drop:
> + flags |= location.NETWORK_FLAG_DROP
> +
> if not flags:
> raise ValueError(_("You must at least pass one flag"))
>
> @@ -551,7 +564,7 @@ class CLI(object):
> asns.append(object)
>
> elif location.country_code_is_valid(object) \
> - or object in ("A1", "A2", "A3"):
> + or object in ("A1", "A2", "A3", "XD"):
> countries.append(object)
>
> else:
> @@ -560,7 +573,7 @@ class CLI(object):
>
> # Default to exporting all countries
> if not countries and not asns:
> - countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
> + countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
>
> # Select the output format
> writer = self.__get_output_formatter(ns)
> diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
> index 5b72be9..5dd4ec6 100644
> --- a/src/python/locationmodule.c
> +++ b/src/python/locationmodule.c
> @@ -1,7 +1,7 @@
> /*
> libloc - A library to determine the location of someone on the Internet
>
> - Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
> + Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>
>
> This library is free software; you can redistribute it and/or
> modify it under the terms of the GNU Lesser General Public
> @@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) {
> if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST))
> return NULL;
>
> + if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP))
> + return NULL;
> +
> // Add latest database version
> if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST))
> return NULL;
> --
> 2.26.2
@@ -10,7 +10,7 @@ location - Query the location database
`location list-countries [--show-name] [--show-continent]`
`location list-networks-by-as ASN`
`location list-networks-by-cc COUNTRY_CODE`
-`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
+`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast|--drop]`
`location lookup ADDRESS [ADDRESS...]`
`location search-as STRING`
`location update [--cron=daily|weekly|monthly]`
@@ -73,7 +73,7 @@ or countries.
+
See above for usage of the '--family' and '--format' parameters.
-'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast]'::
+'list-networks-by-flags [--family=[ipv6|ipv4]] [--format=FORMAT] [--anonymous-proxy|--satellite-provider|--anycast|--drop]'::
Lists all networks that have a certain flag.
+
See above for usage of the '--family' and '--format' parameters.
@@ -152,6 +152,9 @@ msgstr ""
msgid "Anycasts"
msgstr ""
+msgid "Hostile Networks safe to drop"
+msgstr ""
+
msgid "Lists all countries"
msgstr ""
@@ -1,7 +1,7 @@
/*
libloc - A library to determine the location of someone on the Internet
- Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+ Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -27,6 +27,7 @@ enum loc_network_flags {
LOC_NETWORK_FLAG_ANONYMOUS_PROXY = (1 << 0), // A1
LOC_NETWORK_FLAG_SATELLITE_PROVIDER = (1 << 1), // A2
LOC_NETWORK_FLAG_ANYCAST = (1 << 2), // A3
+ LOC_NETWORK_FLAG_DROP = (1 << 3), // XD
};
struct loc_network;
@@ -198,6 +198,8 @@ lookup_network_has_flag(db, address, flag)
iv |= LOC_NETWORK_FLAG_SATELLITE_PROVIDER;
else if (strcmp("LOC_NETWORK_FLAG_ANYCAST", flag) == 0)
iv |= LOC_NETWORK_FLAG_ANYCAST;
+ else if (strcmp("LOC_NETWORK_FLAG_DROP", flag) == 0)
+ iv |= LOC_NETWORK_FLAG_DROP;
else
croak("Invalid flag");
@@ -3,7 +3,7 @@
# #
# libloc - A library to determine the location of someone on the Internet #
# #
-# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
+# Copyright (C) 2020-2021 IPFire Development Team <info@ipfire.org> #
# #
# This library is free software; you can redistribute it and/or #
# modify it under the terms of the GNU Lesser General Public #
@@ -33,6 +33,7 @@ FLAGS = {
_location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
_location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
_location.NETWORK_FLAG_ANYCAST : "A3",
+ _location.NETWORK_FLAG_DROP : "XD",
}
class OutputWriter(object):
@@ -182,6 +182,7 @@ class CLI(object):
);
CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
ON autnum_overrides(number);
+ ALTER TABLE autnum_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
CREATE TABLE IF NOT EXISTS network_overrides(
network inet NOT NULL,
@@ -194,6 +195,7 @@ class CLI(object):
ON network_overrides(network);
CREATE INDEX IF NOT EXISTS network_overrides_search
ON network_overrides USING GIST(network inet_ops);
+ ALTER TABLE network_overrides ADD COLUMN IF NOT EXISTS is_drop boolean;
""")
return db
@@ -301,7 +303,20 @@ class CLI(object):
WHERE networks.autnum = overrides.number
),
FALSE
- ) AS is_anycast
+ ) AS is_anycast,
+ COALESCE(
+ (
+ SELECT is_drop FROM network_overrides overrides
+ WHERE networks.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_drop FROM autnum_overrides overrides
+ WHERE networks.autnum = overrides.number
+ ),
+ FALSE
+ ) AS is_drop
FROM (
SELECT
known_networks.network AS network,
@@ -350,6 +365,9 @@ class CLI(object):
if row.is_anycast:
network.set_flag(location.NETWORK_FLAG_ANYCAST)
+ if row.is_drop:
+ network.set_flag(location.NETWORK_FLAG_DROP)
+
# Add all countries
log.info("Writing countries...")
rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
@@ -966,14 +984,16 @@ class CLI(object):
country,
is_anonymous_proxy,
is_satellite_provider,
- is_anycast
- ) VALUES (%s, %s, %s, %s, %s)
+ is_anycast,
+ is_drop
+ ) VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (network) DO NOTHING""",
"%s" % network,
block.get("country"),
self._parse_bool(block, "is-anonymous-proxy"),
self._parse_bool(block, "is-satellite-provider"),
self._parse_bool(block, "is-anycast"),
+ self._parse_bool(block, "drop"),
)
elif type == "aut-num":
@@ -994,8 +1014,9 @@ class CLI(object):
country,
is_anonymous_proxy,
is_satellite_provider,
- is_anycast
- ) VALUES(%s, %s, %s, %s, %s, %s)
+ is_anycast,
+ is_drop
+ ) VALUES(%s, %s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING""",
autnum,
block.get("name"),
@@ -1003,6 +1024,7 @@ class CLI(object):
self._parse_bool(block, "is-anonymous-proxy"),
self._parse_bool(block, "is-satellite-provider"),
self._parse_bool(block, "is-anycast"),
+ self._parse_bool(block, "drop"),
)
else:
@@ -3,7 +3,7 @@
# #
# libloc - A library to determine the location of someone on the Internet #
# #
-# Copyright (C) 2017 IPFire Development Team <info@ipfire.org> #
+# Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org> #
# #
# This library is free software; you can redistribute it and/or #
# modify it under the terms of the GNU Lesser General Public #
@@ -146,6 +146,9 @@ class CLI(object):
list_networks_by_flags.add_argument("--anycast",
action="store_true", help=_("Anycasts"),
)
+ list_networks_by_flags.add_argument("--drop",
+ action="store_true", help=_("Hostile Networks safe to drop"),
+ )
list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
list_networks_by_flags.add_argument("--format",
choices=location.export.formats.keys(), default="list")
@@ -305,6 +308,12 @@ class CLI(object):
_("Anycast"), _("yes"),
))
+ # Hostile Network
+ if network.has_flag(location.NETWORK_FLAG_DROP):
+ print(format % (
+ _("Hostile Network safe to drop"), _("yes"),
+ ))
+
return ret
def handle_dump(self, db, ns):
@@ -346,6 +355,7 @@ class CLI(object):
location.NETWORK_FLAG_ANONYMOUS_PROXY : "is-anonymous-proxy:",
location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
location.NETWORK_FLAG_ANYCAST : "is-anycast:",
+ location.NETWORK_FLAG_DROP : "drop:",
}
# Iterate over all networks
@@ -523,6 +533,9 @@ class CLI(object):
if ns.anycast:
flags |= location.NETWORK_FLAG_ANYCAST
+ if ns.drop:
+ flags |= location.NETWORK_FLAG_DROP
+
if not flags:
raise ValueError(_("You must at least pass one flag"))
@@ -551,7 +564,7 @@ class CLI(object):
asns.append(object)
elif location.country_code_is_valid(object) \
- or object in ("A1", "A2", "A3"):
+ or object in ("A1", "A2", "A3", "XD"):
countries.append(object)
else:
@@ -560,7 +573,7 @@ class CLI(object):
# Default to exporting all countries
if not countries and not asns:
- countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
+ countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
# Select the output format
writer = self.__get_output_formatter(ns)
@@ -1,7 +1,7 @@
/*
libloc - A library to determine the location of someone on the Internet
- Copyright (C) 2017 IPFire Development Team <info@ipfire.org>
+ Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
@@ -169,6 +169,9 @@ PyMODINIT_FUNC PyInit__location(void) {
if (PyModule_AddIntConstant(m, "NETWORK_FLAG_ANYCAST", LOC_NETWORK_FLAG_ANYCAST))
return NULL;
+ if (PyModule_AddIntConstant(m, "NETWORK_FLAG_DROP", LOC_NETWORK_FLAG_DROP))
+ return NULL;
+
// Add latest database version
if (PyModule_AddIntConstant(m, "DATABASE_VERSION_LATEST", LOC_DATABASE_VERSION_LATEST))
return NULL;