new file mode 100755
@@ -0,0 +1,1382 @@
+#! /usr/bin/perl
+
+############################################################################
+# #
+# IP Address blocklists for IPFire #
+# #
+# This is free software; you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation; either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with IPFire; if not, write to the Free Software #
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+# #
+# Copyright (C) 2018 - 2020 The IPFire team #
+# #
+############################################################################
+# #
+# This script uses a file containing blacklist details in #
+# /var/ipfire/ipblacklist/sources as well as #
+# /var/ipfire/ipblacklistsettings containing an enable/disable flag for #
+# each source. #
+# #
+# Two IPTables chains are used: BLACKLISTIN and BLACKLISTOUT are inserted #
+# inserted into the main INPUT, OUTPUT and FORWARD chains; they capture #
+# packets other than for the ICMP protocol. #
+# #
+# For each blacklist that is loaded, a chain is created to optionally log #
+# and then to drop matching packets. An IPSet is created containing the #
+# addresses or networks blocked by the blacklist, and then rules are added #
+# to the BLACKLISTIN and BLACKLISTOUT chains to jump to this chain if a #
+# packet list matches the set. #
+# #
+# When checking for updates, the modification time is used for each source #
+# and if necessary the list is downloaded. The downloaded list is #
+# compared to the existing IPSet contents and entries created or deleted #
+# as necessary. #
+# #
+############################################################################
+
+use strict;
+#use warnings;
+
+use Carp;
+use Sys::Syslog qw(:standard :macros);
+use HTTP::Request;
+use LWP::UserAgent;
+
+require "/var/ipfire/general-functions.pl";
+
+############################################################################
+# Configuration variables
+#
+# These variables give the locations of various files used by this script
+############################################################################
+
+my $settingsdir = "/var/ipfire/ipblacklist";
+my $savedir = "/var/lib/ipblacklist";
+my $tmpdir = "/var/tmp";
+
+my $settings = "$settingsdir/settings";
+my $sources = "$settingsdir/sources";
+my $checked = "$settingsdir/checked";
+my $modified = "$settingsdir/modified";
+my $iptables_list = "/var/tmp/iptables.txt";
+my $getipstat = "/usr/local/bin/getipstat";
+my $iptables = "/sbin/iptables";
+my $ipset = "/usr/sbin/ipset";
+my $fcrontab = "/usr/bin/fcrontab";
+my $lockfile = "/var/run/ipblacklist.pid";
+my $proxy_settings = "${General::swroot}/proxy/settings";
+my $red_setting = "/var/ipfire/red/iface";
+my $detailed_log = "$tmpdir/ipblacklist_log.txt";
+my $active = "/var/ipfire/red/active";
+
+# Other configuration items
+
+my $margin = 30; # Scheduling allowance for run time etc in seconds
+my $count = 30; # Maximum time to wait for another instance (300s)
+my $max_dl_fails = 3; # Ignore check rate limit for this number of failures
+my $max_size_fraction = 0.7; # Maximum fill fraction of IPSet before enlarging.
+my $min_ipset_entries = 1024; # The minimum size of an IPSet.
+my $max_dl_bytes = 10_485_760; # Maximum number of bytes to download.
+my %parsers = ( 'ip-or-net-list' => \&parse_ip_or_net_list,
+ 'dshield' => \&parse_dshield );
+
+############################################################################
+# Default settings
+# Should be overwritten by reading settings files
+############################################################################
+
+my %sources = ( );
+
+my %settings = ( 'DEBUG' => 0,
+ 'LOGGING' => 'on',
+ 'ENABLE' => 'off' );
+
+my %proxy_settings = ( 'UPSTREAM_PROXY' => '' ); # No Proxy in use
+
+############################################################################
+# Function prototypes
+############################################################################
+
+sub abort( $ );
+sub create_list( $ );
+sub create_ipset( $$$ );
+sub debug( $$ );
+sub delete_list( $ );
+sub disable_logging();
+sub disable_updates();
+sub do_delete();
+sub do_start();
+sub do_stop();
+sub do_update();
+sub download_list( $$$ );
+sub download_check_header_time( $$$ );
+sub download_wget( $$$ );
+sub enable_logging();
+sub enable_updates();
+sub get_ipsets();
+sub get_rate_seconds( $ );
+sub iptables( $ );
+sub ipset( $ );
+sub stop_ipset();
+sub is_connected();
+sub log_message( $$ );
+sub parse_dshield( $ );
+sub parse_ip_or_net_list( $ );
+sub read_ipset( $$$$ );
+sub update_list( $$$ );
+
+############################################################################
+# Variables
+############################################################################
+
+my %chains; # The Blacklist IPSets already loaded
+my %old_blacklist; # Already blocked IP Addresses and/or networks
+ # downloaded for current blacklist
+my $update_status = 0; # Set to 1 to update status file
+my $ipset_running = 0; # Set to 1 if IPSet process is running
+my %status; # Status information
+my %checked; # Time blacklists last changed
+my %modified; # Time blacklists last modified
+my $red_iface; # The name of the red interface
+
+############################################################################
+# Synchronise runs
+############################################################################
+
+# This script can be triggered either by cron or the WUI. If another
+# instance is running, wait for it to finish or timeout.
+
+while (-r $lockfile and $count > 0)
+{
+ open LOCKFILE, '<', $lockfile or (abort "Can't open lockfile", last);
+ my $pid = <LOCKFILE>;
+ close LOCKFILE;
+
+ chomp $pid;
+
+ last unless (-e "/proc/$pid");
+
+ sleep 10;
+ $count--;
+}
+
+# Create pid file before starting main processing
+
+open LOCKFILE, '>', '/var/run/ipblacklist.pid' or abort "Can't open PID file: $!";
+print LOCKFILE "$$\n";
+close LOCKFILE;
+
+############################################################################
+# Set up for update
+############################################################################
+
+mkdir $settingsdir unless (-d $settingsdir);
+
+# Connect to the system log
+
+openlog( "ipblacklist", "nofatal", LOG_USER);
+log_message LOG_INFO, "Starting IP Blacklist processing";
+
+# Read settings
+
+General::readhash( $settings, \%settings ) if (-e $settings);
+General::readhash( $checked, \%checked ) if (-e $checked);
+General::readhash( $modified, \%modified ) if (-e $modified);
+General::readhash( $proxy_settings, \%proxy_settings ) if (-e $proxy_settings);
+
+if (-r $sources)
+{
+ debug 1, "Reading sources file";
+
+ eval qx|/bin/cat $sources|;
+}
+
+if (-r $red_setting)
+{
+ open REDIF, '<', $red_setting or (abort "Can't open red interface file", exit);
+ $red_iface = <REDIF>;
+ chomp $red_iface;
+ close REDIF;
+}
+
+if (@ARGV)
+{
+ foreach my $cmd (@ARGV)
+ {
+ if ('update' =~ m/^$cmd/i)
+ {
+ # Called hourly when enabled and on setting changes.
+ # Update the blacklists.
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ do_update();
+ }
+ }
+ elsif ('start' =~ m/^$cmd/i)
+ {
+ # Called during system startup.
+ # Restore saved blacklists.
+ # Don't do an update since can take too long.
+
+ do_start() if ($settings{'ENABLE'} eq 'on');
+ }
+ elsif ('stop' =~ m/^$cmd/i)
+ {
+ # Called when shutting down.
+ # Delete IPSets and IPTables chains
+
+ do_stop();
+ }
+ elsif ('restore' =~ m/^$cmd/i)
+ {
+ # Called after restoring backup.
+ # Delete IPSets and IPTables chains, then re-create them with the new
+ # (restored) settings and make sure updates are enabled.
+
+ do_stop();
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ do_start();
+ enable_updates();
+ }
+ else
+ {
+ disable_updates();
+ }
+ }
+ elsif ('log-on' =~ m/^$cmd/i)
+ {
+ # Called from WUI.
+ # Create entries in IPTables chains to log dropped packets.
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ enable_logging();
+ }
+ }
+ elsif ('log-off' =~ m/^$cmd/i)
+ {
+ # Called from WUI
+ # Delete entries in IPTables chains to log dropped packets.
+
+ disable_logging();
+ }
+ elsif ('enable' =~ m/^$cmd/i)
+ {
+ # Called from WUI to enable blacklists
+ # Do an update and then enable automatic updates
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ do_update();
+ enable_updates();
+ }
+ }
+ elsif ('disable' =~ m/^$cmd/i)
+ {
+ # Called from WUI to disable blacklists.
+ # Disable updates, delete IPSets, IPTables chains and save files.
+
+ disable_updates();
+ do_delete();
+ }
+ else
+ {
+ print "Usage: $0 [update|start|stop|restart|log-on|log-off|enable|disable]\n";
+ }
+ }
+}
+elsif ($settings{'ENABLE'} eq 'on') # Default action if none specified
+{
+ do_update();
+}
+
+stop_ipset();
+
+if ($update_status)
+{
+ debug 1, "Writing updated status file";
+
+ General::writehash( $checked, \%checked );
+ General::writehash( $modified, \%modified );
+}
+
+# Remove the pid file
+
+unlink $lockfile;
+
+log_message LOG_INFO, "Finished IP Blacklist processing";
+closelog();
+
+
+#------------------------------------------------------------------------------
+# sub do_stop
+#
+# Deletes all the IPTables chains and the IPSets
+#------------------------------------------------------------------------------
+
+sub do_stop()
+{
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Stopping IP Blacklists";
+
+ foreach my $list ( sort keys %sources )
+ {
+ delete_list( $list ) if (exists $chains{$list});
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_start
+#
+# Recreates the IPTables chains and the IPSets from the saved values
+#------------------------------------------------------------------------------
+
+sub do_start()
+{
+ log_message LOG_NOTICE, "Starting IP Blacklists";
+
+ foreach my $list ( sort keys %sources )
+ {
+ delete_list( $list ) if (exists $chains{$list}); # Make sure OK to start
+
+ if ((-e "$savedir/$list.conf") and ($red_iface))
+ {
+ log_message LOG_INFO, "Restoring blacklist $list";
+ system( "$ipset restore -f $savedir/$list.conf" ); # Can't use the ipset
+ # function to do this
+
+ create_list( $list );
+ }
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_delete
+#
+# Deletes the IPTables chains, the IPSets and the saved values.
+#------------------------------------------------------------------------------
+
+sub do_delete()
+{
+ # Get the list of current ipsets
+
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Deleting IP Blacklists";
+
+ foreach my $source ( sort keys %sources )
+ {
+ if (exists $chains{$source})
+ {
+ delete_list( $source );
+ }
+
+ if (-e "$savedir/$source.conf")
+ {
+ unlink "$savedir/$source.conf";
+ }
+ }
+
+ %modified = ();
+ $update_status = 1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_update
+#
+# Updates all the blacklists.
+# Creates or deletes the blacklist firewall rules as necessary and checks for
+# updates to the blacklists. Each blacklist has its own minimum elapsed time
+# between updates, which is specified in the sources file, so the time of each
+# check is stored.
+#------------------------------------------------------------------------------
+
+sub do_update()
+{
+ return unless ($red_iface);
+
+ # Get the list of current ipsets
+
+ get_ipsets();
+
+ # Check sources
+
+ debug 1, "Checking blacklist sources";
+
+ LIST:
+ foreach my $list ( sort keys %sources )
+ {
+ my @new_blacklist = ();
+ my $name = $sources{$list}{'name'};
+ my $last_checked = $checked{$list} || 0;
+ my $failures = $checked{"${list}_failures"} || 0;
+ my $enabled = 0;
+
+ if (exists $settings{$list})
+ {
+ $enabled = $settings{$list} eq 'on';
+ }
+
+ if ($enabled and is_connected())
+ {
+ debug 1, "Checking blacklist source: $name";
+
+ # Calculate the per list rate
+
+ my $rate = get_rate_seconds( $sources{$list}{'rate'} );
+
+ # Has enough time passed since the last time we checked the list?
+ # Ignore the limit if the last download failed
+
+ if (($last_checked + $rate) < (time() + $margin) or
+ ($failures > 0 and $failures < $max_dl_fails))
+ {
+ my $type = 'hash:ip';
+
+ download_list( $list, \@new_blacklist, \$type );
+
+ next LIST unless (@new_blacklist);
+
+ if (not exists $chains{$list})
+ {
+ # Doesn't currently exist: Create it.
+
+ create_ipset( $list, $type, scalar @new_blacklist );
+ create_list( $list );
+ }
+
+ update_list( $list, \@new_blacklist, $type );
+ }
+ }
+ elsif (exists $chains{$list})
+ {
+ # Exists, but not enabled: Delete it.
+
+ delete_list( $list );
+
+ # Delete the save file
+ # Don't delete the checked time from the status, in case the list is
+ # re-enabled quickly - don't want to exceed maximum allowed download
+ # rate.
+
+ unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+ delete $modified{$list} if (exists $modified{$list});
+ delete $checked{"${list}_failures"} if (exists $checked{"${list}_failures"});
+ $update_status = 1;
+ }
+ }
+
+ # Check for any lists that don't exist any more
+
+ foreach my $list (keys %modified)
+ {
+ next if (exists $sources{$list});
+
+ delete_list( $list );
+
+ # Delete the save file
+
+ unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+ # Delete from the status
+
+ delete $modified{$list} if (exists $modified{$list});
+ $update_status = 1;
+ }
+
+ foreach my $list (keys %checked)
+ {
+ next if ($list =~ m/_failures/);
+ next if (exists $sources{$list});
+
+ delete $checked{$list};
+ delete $checked{"${list}_failures"};
+ delete $settings{$list} if (exists $settings{$list});
+ $update_status = 1;
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub get_rate_seconds( text )
+#
+# Converts a check rate into seconds. A sanity check is made on the coverted
+# value.
+#
+# Parameters:
+# text The value to convert in the form nnnu, where nnn is a number and u
+# is either m (minutes), h (hours) or d (days). Hours is assumed if
+# not specified and everything after the first letter is ignored.
+#------------------------------------------------------------------------------
+
+sub get_rate_seconds( $ )
+{
+ my ($text) = @_;
+
+ my ($value, $unit) = (uc $text) =~ m/(\d+)([DHM]?)/;
+
+ if ($unit eq 'D') # Days
+ {
+ $value *= 60 * 60 * 24;
+ }
+ elsif ($unit eq 'M') # Minutes
+ {
+ $value *= 60;
+ }
+ else # Everything else - assume hours
+ {
+ $value *= 60 * 60;
+ }
+
+ # Sanity check - limit to range 5 min .. 1 week
+
+ # d h m s
+ $value = 5 * 60 if ($value < 5 * 60);
+ $value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60);
+
+ return $value;
+}
+
+
+#------------------------------------------------------------------------------
+# sub is_connected()
+#
+# Checks that the system is connected to the internet.
+#
+# This looks for a file created by IPFire when connected to the internet
+#------------------------------------------------------------------------------
+
+sub is_connected()
+{
+ return (-e $active);
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_list( list )
+#
+# Creates a new IPTables chain for a blacklist source.
+# The set must be created before calling this function. Two rules are added to
+# the chain:
+# (optional) 1 Log the packet
+# 2 Drop the packet
+#
+# The log rule is only added when logging is enabled by the WUI.
+#
+# Rules are then added to the BLACKLISTIN and BLACKLISTOUT chains that check
+# the packet's IP address against the IPSet and then jump to the newly created
+# chain.
+#
+# Parameters:
+# list The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub create_list( $ )
+{
+ my ($list) = @_;
+
+ log_message LOG_INFO, "Create IPTables chains for blacklist $list";
+
+ # Create new chain in filter table
+
+ iptables( "-N ${list}_DROP" ) == 0 or
+ ( abort "Could not create IPTables chain ${list}_DROP", return );
+
+ # Add the logging and drop rules
+
+ if ($settings{'LOGGING'} eq 'on')
+ {
+ iptables( "-A ${list}_DROP -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" ) == 0 or
+ ( abort "Could not create IPTables chain $list LOG rule", return );
+ }
+
+ iptables( "-A ${list}_DROP -j DROP" ) == 0 or
+ ( abort "Could not create IPTables chain $list drop rule", return );
+
+ # Add the rules to check against the set
+
+ iptables( "-A BLACKLISTIN -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" );
+ iptables( "-A BLACKLISTOUT -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub delete_list( $list )
+#
+# Deletes an IPTables chain when a blacklist source is disabled. Also flushes
+# and destroys the IPSet.
+#
+# Parameters:
+# list The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub delete_list( $ )
+{
+ my ($list) = @_;
+
+ log_message LOG_INFO, "Delete IPTables chains for blacklist $list";
+
+ # Remove the blacklist chains from the main INPUT and OUTPUT chains
+
+ iptables( "-D BLACKLISTIN -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" );
+ iptables( "-D BLACKLISTOUT -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" );
+
+ # Flush and delete the chain
+
+ iptables( "-F ${list}_DROP" );
+ iptables( "-X ${list}_DROP" );
+
+ # Flush and delete the set
+
+ ipset( "flush $list" );
+ ipset( "destroy $list" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_list( list, ref_list, ref_type )
+#
+# Downloads the IP Addresses for a blacklist.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+# list The name of the blacklist
+# ref_list A reference to an array to store the downloaded blacklist
+# ref_type A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_list( $$$ )
+{
+ my ($list, $new_blacklist, $type) = @_;
+
+ $checked{$list} = time(); # Record that the list has been checked
+ $update_status = 1;
+
+ # Check the parser for the blacklist
+
+ if (not exists $parsers{ $sources{$list}{'parser'} })
+ {
+ log_message LOG_ERR, "Can't find parser $sources{$list}{'parser'} for $list blacklist";
+ return;
+ }
+
+ # Add alternative download mechanisms here
+
+ download_check_header_time( $list, $new_blacklist, $type );
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_check_header_time( list, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist. The If-Modified-Since header is
+# specified in the request so that only updated lists are downloaded (providing
+# that the server supports this functionality).
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+# list The name of the blacklist
+# ref_list A reference to an array to store the downloaded blacklist
+# ref_type A reference to store the type of the blacklist
+#
+# Returns:
+# The list type: 'hash:ip' or 'hash:net'
+#------------------------------------------------------------------------------
+
+sub download_check_header_time( $$$ )
+{
+ my ($list, $new_blacklist, $type) = @_;
+ my $found_ip = 0;
+ my $found_net = 0;
+
+ # Get the parser for the blacklist
+
+ my $parser = $parsers{ $sources{$list}{'parser'} };
+
+ debug 1, "Checking for blacklist $list updates with LWP";
+
+ # Create a user agent for downloading the blacklist
+ # Limit the download size for safety
+
+ my $ua = LWP::UserAgent->new( max_size => $max_dl_bytes );
+
+ # Get the Proxy settings
+
+ if ($proxy_settings{'UPSTREAM_PROXY'})
+ {
+ if ($proxy_settings{'UPSTREAM_USER'})
+ {
+ $ua->proxy( [["http", "https"] => "http://$proxy_settings{'UPSTREAM_USER'}:$proxy_settings{'UPSTREAM_PASSWORD'}\@$proxy_settings{'UPSTREAM_PROXY'}/"] );
+ }
+ else
+ {
+ $ua->proxy( [["http", "https"] => "http://$proxy_settings{'UPSTREAM_PROXY'}/"] );
+ }
+ }
+
+ # Get the last modified time
+
+ my $modified = gmtime( $modified{$list} || 0 );
+
+ # Download the blacklist
+
+ my $response = $ua->get( $sources{$list}{'url'}, 'If-Modified-Since' => $modified );
+
+ if (not $response->is_success)
+ {
+ if ($response->code == 304)
+ {
+ # Not an error - list has not been modified
+ debug 1, "Blacklist $list not modified";
+
+ return;
+ }
+
+ log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: ". $response->status_line;
+ $checked{"${list}_failures"}++;
+
+ return;
+ }
+
+ $modified{$list} = $response->last_modified;
+ $checked{"${list}_failures"} = 0;
+
+ # Parse the downloaded list, checking if it's a list of addresses or nets
+
+ foreach my $line (split /[\r\n]+/, $response->content)
+ {
+ chomp $line;
+
+ my $address = &$parser( $line );
+
+ next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);
+
+ if ($address =~ m|/32|)
+ {
+ $address =~ s|/32||;
+ $found_ip = 1;
+ }
+ elsif ($address =~ m|/\d+|)
+ {
+ $found_net = 1;
+ }
+ else
+ {
+ $found_ip = 1;
+ }
+
+ push @{ $new_blacklist }, $address;
+ }
+
+ if ($found_net and $found_ip)
+ {
+ # Convert mixed addresses and networks to all networks
+
+ foreach my $address (@{ $new_blacklist })
+ {
+ $address .= '/32' unless ($address =~ m|/\d+|);
+ }
+
+ $found_ip = 0;
+ }
+
+ $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub read_ipset( list, old, type, maxelem )
+#
+# Reads the existing contents and type of the set.
+#
+# Parameters:
+# list The name of the blacklist
+# old Reference to array to contain blacklist
+# type Reference to type
+# maxelem Reference to maximum number of elements
+#------------------------------------------------------------------------------
+
+sub read_ipset( $$$$ )
+{
+ my ($list, $old, $type, $maxelem) = @_;
+ my $found_net = 0;
+ my $found_ip = 0;
+
+ debug 2, "Reading existing ipset for blacklist $list";
+
+ foreach my $line (qx/$ipset list $list/)
+ {
+ if ($line =~ m|Header:.*maxelem (\d+)|)
+ {
+ $$maxelem = $1;
+ next;
+ }
+
+ next unless ($line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|);
+
+ my $address = $1;
+
+ if ($address =~ m|/32|)
+ {
+ $found_ip = 1;
+ $address =~ s|/32$||;
+ }
+ elsif ($address =~ m|/\d+$|)
+ {
+ $found_net = 1;
+ }
+ else
+ {
+ $found_ip = 1;
+ }
+
+ $$old{$address} = 1;
+ }
+
+ if ($found_ip and $found_net)
+ {
+ # Convert mixed addresses and networks to all networks
+
+ my @ads_list = keys %{ $old };
+
+ foreach my $address (@ads_list)
+ {
+ unless ($address =~ m|/\d+|)
+ {
+ delete $$old{$address};
+ $$old{"$address/32"} = 1;
+ }
+ }
+
+ $found_ip = 0;
+ }
+
+ $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub update_list( list, new, new_type )
+#
+# Updates the IP Addresses for a blacklist
+#
+# The new list is compared to the existing list and new entries added or old
+# entries deleted as necessary. If the list type ('hash:ip' or 'hash:net') has
+# changed then the IPSet is deleted and re-created with the new type.
+#
+# Parameters:
+# list The name of the blacklist
+# new Reference to array of new blacklist entries
+# new_type The type of the updated list (hash:ip or hash:net)
+#------------------------------------------------------------------------------
+
+sub update_list( $$$ )
+{
+ my ($list, $new, $new_type) = @_;
+ my %old;
+ my $old_type;
+ my $changes = 0;
+ my $maxelem = 0;
+
+ debug 1, "Checking for $list blacklist update from $sources{$list}{'url'}";
+
+ if (exists $chains{$list} )
+ {
+ my $recreate_ipset = 0;
+
+ read_ipset( $list, \%old, \$old_type, \$maxelem );
+
+ # Check the IPSet type hasn't changed
+
+ if ($new_type ne $old_type)
+ {
+ log_message LOG_NOTICE, "Blacklist $list changed type from $old_type to $new_type";
+ $recreate_ipset = 1;
+ }
+
+ if ($max_size_fraction * $maxelem < scalar @{ $new } )
+ {
+ log_message LOG_NOTICE, "Blacklist $list changed size from $maxelem";
+ $recreate_ipset = 1;
+ }
+
+ if ($recreate_ipset)
+ {
+ # Change the IPSet type and/or size. This requires removing references
+ # to it first. We could delete and then create the chain, but doing it
+ # like this keeps the statistics.
+
+ # Remove the IPSet from the IPTables chains
+
+ iptables( "-D 'BLACKLISTIN' -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ) == 0 or
+ log_message LOG_ERR, "Could not remove ${list} from BLACKLISTIN chain";
+
+ iptables( "-D 'BLACKLISTOUT' -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ) == 0 or
+ log_message LOG_ERR, "Could not remove ${list} from BLACKLISTOUT chain";
+
+ # Flush and delete the old set
+
+ ipset( "flush $list" );
+ ipset( "destroy $list" );
+
+ %old = (); # Since we've deleted the old set it can't have any entries.
+
+ # Create the new ipset
+
+ create_ipset( $list, $new_type, scalar @{ $new } );
+
+ # Add the rules to check against the set
+
+ iptables( "-A 'BLACKLISTIN' -p ALL -i $red_iface -m set --match-set $list src -j ${list}_DROP" ) == 0 or
+ log_message LOG_ERR, "Could not add IPSet $list to BLACKLISTIN chain";
+
+ iptables( "-A 'BLACKLISTOUT' -p ALL -o $red_iface -m set --match-set $list dst -j ${list}_DROP" ) == 0 or
+ log_message LOG_ERR, "Could not add IPSet $list to BLACKLISTOUT chain";
+ }
+ }
+
+ # Process the blacklist
+
+ foreach my $address ( @{ $new } )
+ {
+ # We've got an address. Add to IPSet if it's new
+
+ if (exists $old{$address})
+ {
+ delete $old{$address}; # Not new - don't delete from chain later
+ }
+ else
+ {
+ ipset( "add $list $address -exist" ); # New - add it
+
+ $changes++;
+ }
+
+ debug 3, "Add net $address to blacklist $list";
+ }
+
+ # Delete old entries that aren't needed any more
+
+ debug 2, "Removing deleted rules from IPTables chain for blacklist $list";
+
+ foreach my $address ( keys %old )
+ {
+ ipset( "del $list $address" );
+
+ $changes++;
+
+ debug 3, "Delete old net $address from blacklist $list";
+ }
+
+ log_message LOG_INFO, "Updated $list blacklist with $changes changes";
+
+ # Save the blacklist for the next reboot
+
+ mkdir "$savedir" unless (-d "$savedir" );
+
+ ipset( "save $list -file $savedir/$list.conf" ) if ($changes > 0);
+
+ stop_ipset();
+}
+
+
+#------------------------------------------------------------------------------
+# sub get_ipsets( )
+#
+# Gets a list of the current IPSets
+#------------------------------------------------------------------------------
+
+sub get_ipsets( )
+{
+ debug 1, "Reading list of existing ipsets";
+
+ my @sets = qx($ipset -n list);
+
+ # Parse the tables
+
+ foreach my $line (@sets)
+ {
+ chomp $line;
+
+ next unless ($line);
+
+ $chains{$line} = 1;
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_ipset( list, type, size )
+#
+# Creates a new IPSet. The current size of the set is determined by taking the
+# next power of two greater than the number of entries; the maximum size is set
+# to double this, subject to a minimum size. This allows for future expansion.
+#
+# Parameters:
+# list The name of the blacklist
+# type The type of the blacklist (hash:ip or hash:net)
+# size The number of entries in the lsit
+#------------------------------------------------------------------------------
+
+sub create_ipset( $$$ )
+{
+ my ($list, $type, $size) = @_;
+
+ my $hashsize = 1;
+ $hashsize <<= 1 while ($hashsize < $size);
+ my $maxsize = $hashsize * 2;
+ $maxsize = $min_ipset_entries if ($maxsize < $min_ipset_entries);
+
+ # Create the new ipset
+ ipset( "create $list $type hashsize $hashsize maxelem $maxsize" );
+ stop_ipset(); # Need to do this to action the IPSet commands
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_logging()
+#
+# Enable logging of packets dropped by IP Blacklist rules.
+# This adds a rule to log the packet to each lists' IPTables chain.
+#------------------------------------------------------------------------------
+
+sub enable_logging()
+{
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Enabling IP Blacklist logging";
+
+ foreach my $list ( sort keys %sources )
+ {
+ if (exists $chains{$list})
+ {
+ iptables( "-I ${list}_DROP 1 -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" );
+ }
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_logging()
+#
+# Disable logging of packets dropped by IP Blacklist rules.
+# This deletes a rule to log the packet from each lists' IPTables chain.
+#------------------------------------------------------------------------------
+
+sub disable_logging()
+{
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Disabling IP Blacklist logging";
+
+ foreach my $list ( sort keys %sources )
+ {
+ if (exists $chains{$list})
+ {
+ iptables( "-D ${list}_DROP -j LOG -m limit --limit 10/second --log-prefix 'BLKLST_$list'" );
+ }
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_updates()
+#
+# Adds a command to the fcrontab to run the update hourly.
+# If there is a command already in the fcrontab to do this it will be
+# uncommented, otherwise a new line is added.
+#
+# The update is executed at an offset from the hour so that all the users don't
+# try to download the updates at exactly the same time - the blacklists are
+# provided free, so it's good manners to spread the load on the servers. The
+# offset is initialised to a random number that avoids running on the hour
+# (when a lot of other things happen), and every fifteen minutes thereafter.
+#------------------------------------------------------------------------------
+
+sub enable_updates()
+{
+ my @lines = qx/$fcrontab -l/;
+ my $found = 0;
+
+ # Check for an existing fcrontab entry.
+
+ foreach my $line (@lines)
+ {
+ if ($line =~ m|/usr/local/bin/ipblacklist|)
+ {
+ return if ($line !~ m/^#/); # Already enabled - do nothing
+
+ # Already in fcrontab - uncomment the line
+
+ $line =~ s/^#+//;
+ $found = 1;
+ log_message LOG_INFO, "Enable IP Address Blacklist update in crontab";
+ last;
+ }
+ }
+
+ if (not $found)
+ {
+ # Add a new entry to fcrontab
+
+ my $start = int( rand(13) ) + 1;
+
+ my $times = $start;
+
+ for (my $offset = $times+15 ; $offset < 60 ; $offset += 15)
+ {
+ $times .= ",$offset";
+ }
+
+ push @lines, "\n";
+ push @lines, "# IP Blacklist update\n";
+ push @lines, "\%nice(1) $times * * * * /usr/local/bin/ipblacklist\n";
+ log_message LOG_INFO, "Add IP Address Blacklist update to crontab";
+ }
+
+ open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+ print FCRONTAB @lines;
+ close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_updates()
+#
+# Comments out the entry in the fcrontab that runs the updates.
+#------------------------------------------------------------------------------
+
+sub disable_updates()
+{
+ my @lines = qx/$fcrontab -l/;
+ my $found = 0;
+
+ foreach my $line (@lines)
+ {
+ if ($line =~ m|/usr/local/bin/ipblacklist|)
+ {
+ return if ($line =~ m/^#/); # Already disabled - do nothing
+
+ # In fcrontab - comment the line
+
+ $line =~ s/^#*/#/;
+ $found = 1;
+ log_message LOG_INFO, "Disable IP Address Blacklist updates";
+ last;
+ }
+ }
+
+ return unless ($found); # Don't update crontab unnecessarily
+
+ open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+ print FCRONTAB @lines;
+ close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_ip_or_net_list( line )
+#
+# Parses an input line, looking for lines starting with an IP Address or
+# Network specification.
+#
+# Parameters:
+# line The line to parse
+#
+# Returns:
+# Either an IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_ip_or_net_list( $ )
+{
+ my ($line) = @_;
+
+ $line =~ m|^(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;
+
+ return $1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_dshield( line )
+#
+# Parses an input line removing comments.
+#
+# The format is:
+# Start Addrs End Addrs Netmask Nb Attacks Network Name Country email
+# We're only interested in the start address and netmask.
+#
+# Parameters:
+# line The line to parse
+#
+# Returns:
+# Either and IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_dshield( $ )
+{
+ my ($line) = @_;
+
+ return "" if ($line =~ m/^\s*#/);
+
+ $line =~ s/#.*$//;
+
+ # |Start addrs | |End Addrs | |Mask
+ $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)|;
+
+ return unless ($1);
+ return "$1/32" unless ($2);
+
+ return "$1/$2";
+}
+
+
+#------------------------------------------------------------------------------
+# sub iptables( cmd )
+#
+# Executes an IPTables command, waiting for the internal lock to ensure only
+# one change is made at a time.
+#
+# Parameters:
+# cmd The command to execute
+#
+# Returns:
+# Status of command
+#------------------------------------------------------------------------------
+
+sub iptables( $ )
+{
+ my ($cmd) = @_;
+
+ return system( "$iptables $cmd" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub ipset( cmd )
+#
+# Executes an IPSet command. The command is piped to a sub-process running
+# ipset, rather than exected separately. This saves the overhead of starting a
+# new process for each command. The sub-process is started if it's not already
+# running.
+#
+# Note that the pipe is buffered so commands are not necessarily executed
+# immediately. Use ipset_stop() to force commands to be executed. This should
+# be done before relying on anything that the ipset commands do, for example
+# before referencing the IPSet in an IPTables command.
+#
+# Parameters:
+# cmd The command to execute
+#------------------------------------------------------------------------------
+
+sub ipset( $ )
+{
+ my ($cmd) = @_;
+
+ if (not $ipset_running)
+ {
+ local $SIG{PIPE} = 'IGNORE';
+ open IPSET, "|-", $ipset, "restore" or die "Can't start ipset: $!";
+ $ipset_running = 1;
+ }
+
+ print IPSET "$cmd\n";
+}
+
+
+#------------------------------------------------------------------------------
+# sub stop_ipset( )
+#
+# Stops the ipset sub-process.
+# This causes any pending ipset commands to be executed.
+#------------------------------------------------------------------------------
+
+sub stop_ipset( )
+{
+ if ($ipset_running)
+ {
+ close IPSET or abort "ipset process died: $! $?";
+ $ipset_running = 0;
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub abort( message, parameters... )
+#
+# Used when aborting the current activity, printing out an error message.
+#
+# Parameters:
+# message Message to be printed
+#------------------------------------------------------------------------------
+
+sub abort( $ )
+{
+ my ($message) = @_;
+
+ log_message( LOG_ERR, $message );
+ carp $message;
+}
+
+
+#------------------------------------------------------------------------------
+# sub log_message( level, message )
+#
+# Logs a message. If the script is run from a terminal messages are also
+# output on STDOUT.
+#
+# Parameters:
+# level Severity of message
+# message Message to be logged
+#------------------------------------------------------------------------------
+
+sub log_message( $$ )
+{
+ my ($level, $message) = @_;
+
+ print "($level) $message\n" if (-t STDIN);
+ syslog( $level, $message );
+}
+
+
+#------------------------------------------------------------------------------
+# sub debug( level, message )
+#
+# Optionally logs a debug message. If the script is run from a terminal, level
+# 1 debug messages are output regardless of the debug setting.
+#
+# Parameters:
+# level Debug level
+# message Message to be logged
+#------------------------------------------------------------------------------
+
+sub debug( $$ )
+{
+ my ($level, $message) = @_;
+
+ if (($level <= $settings{'DEBUG'}) or
+ ($level == 1 and -t STDIN))
+ {
+ log_message LOG_DEBUG, $message;
+ }
+}