new file mode 100755
@@ -0,0 +1,1558 @@
+#! /usr/bin/perl
+
+############################################################################
+# #
+# IP Address blocklists for IPFire #
+# #
+# This is free software; you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation; either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# This is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with IPFire; if not, write to the Free Software #
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #
+# #
+# Copyright (C) 2018 - 2019 The IPFire team #
+# #
+############################################################################
+# #
+# This script use a file containing blacklist details in #
+# /var/ipfire/ipblacklist/sources as well as #
+# /var/ipfire/ipblacklistsettings containing an enable/disable flag for #
+# each source. #
+# #
+# Two IPTables chains are used, IPBLACKLISTREDIN and IPBLACKLISTREDOUT, #
+# which are inserted into the main INPUT, OUTPUT and FORWARD chains. #
+# #
+# For each blacklist that is loaded, a chain is created to optionally log #
+# and then to drop matching packets. An IPSet is created containing the #
+# addresses or networks blocked by the blacklist, and then rules are added #
+# to the IPBLACKLISTREDIN and IPBLACKLISTREDOUT chains to jump to this #
+# chain if appropriate packet list matches in the set. #
+# #
+# When checking for updates, the modification time is read for each source #
+# and if necessary the list is downloaded. The downloaded list is #
+# compared to the existing IPSet contents and entries created or deleted #
+# as necessary. #
+# #
+############################################################################
+
+use strict;
+use warnings;
+
+use Carp;
+use Sys::Syslog qw(:standard :macros);
+use HTTP::Request;
+use LWP::UserAgent;
+
+require "/var/ipfire/general-functions.pl";
+
+############################################################################
+# Configuration variables
+#
+# These variables give the locations of various files used by this script
+############################################################################
+
+my $settingsdir = "/var/ipfire/ipblacklist";
+my $savedir = "/var/lib/ipblacklist";
+my $tmpdir = "/var/tmp";
+
+my $settings = "$settingsdir/settings";
+my $sources = "$settingsdir/sources";
+my $checked = "$settingsdir/checked";
+my $modified = "$settingsdir/modified";
+my $iptables_list = "/var/tmp/iptables.txt";
+my $getipstat = "/usr/local/bin/getipstat";
+my $iptables = "/sbin/iptables";
+my $ipset = "/usr/sbin/ipset";
+my $fcrontab = "/usr/bin/fcrontab";
+my $lockfile = "/var/run/ipblacklist.pid";
+my $proxy_settings = "${General::swroot}/proxy/settings";
+my $red_setting = "/var/ipfire/red/iface";
+my $detailed_log = "$tmpdir/ipblacklist_log.txt";
+my $autoblacklist = 'AUTOBLACKLIST';
+
+my %parsers = ( 'text-with-hash-comments' => \&parse_text_with_hash_comments,
+ 'text-with-semicolon-comments' => \&parse_text_with_semicolon_comments,
+ 'dshield' => \&parse_dshield );
+
+############################################################################
+# Default settings
+# Should be overwritten by reading settings files
+############################################################################
+
+my %sources = ( );
+
+my %settings = ( 'DEBUG' => 0,
+ 'LOGGING' => 'on',
+ 'RATE' => 24,
+ 'ENABLE' => 'off' );
+
+my %proxy_settings = ( 'UPSTREAM_PROXY' => '' ); # No Proxy in use
+
+############################################################################
+# Function prototypes
+############################################################################
+
+sub abort( $ );
+sub autoblacklist_update();
+sub autoblacklist_clear();
+sub create_autoblacklist();
+sub create_list( $ );
+sub create_ipset( $$$ );
+sub debug( $$ );
+sub delete_autoblacklist();
+sub delete_list( $ );
+sub disable_logging();
+sub disable_updates();
+sub do_delete();
+sub do_start();
+sub do_stop();
+sub do_update();
+sub download_list( $$$ );
+sub download_check_header_time( $$$ );
+sub download_wget( $$$ );
+sub enable_logging();
+sub enable_updates();
+sub get_ipsets();
+sub iptables( $ );
+sub ipset( $ );
+sub stop_ipset();
+sub is_connected();
+sub log_message( $$ );
+sub parse_dshield( $ );
+sub parse_text_with_hash_comments( $ );
+sub parse_text_with_semicolon_comments( $ );
+sub read_ipset( $$$ );
+sub update_list( $$$ );
+
+############################################################################
+# Variables
+############################################################################
+
+my %chains; # The Blacklist IPSets already loaded
+my %old_blacklist; # Already blocked IP Addresses and/or networks
+ # downloaded for current blacklist
+my $update_status = 0; # Set to 1 to update status file
+my $ipset_running = 0; # Set to 1 if IPSet process is running
+my %status; # Status information
+my %checked; # Time blacklists last changed
+my %modified; # Time blacklists last modified
+my $red_iface; # Name of red interface
+my $hours = 3600; # One hour in seconds
+my $margin = 600; # Allowance for run time etc
+my $count = 30; # Maximum time to wait for another instance (300s)
+my @wget_status = ( 'Success', 'Error', 'Parse Error', 'File I/O Error',
+ 'Network Error', 'SSL Verification Error',
+ 'Authentication Error', 'Protocol Error', 'Server Error' );
+
+
+############################################################################
+# Synchronise runs
+############################################################################
+
+# This script can be triggered either by cron or the WUI. If another
+# instance is running, wait for it to finish.
+
+while (-r $lockfile and $count > 0)
+{
+ open LOCKFILE, '<', $lockfile or die "Can't open lockfile";
+ my $pid = <LOCKFILE>;
+ close LOCKFILE;
+
+ chomp $pid;
+
+ last unless (-e "/proc/$pid");
+
+ sleep 10;
+ $count--;
+}
+
+# Create pid file before starting main processing
+
+open LOCKFILE, '>', '/var/run/ipblacklist.pid' or die "Can't open PID file: $!";
+print LOCKFILE "$$\n";
+close LOCKFILE;
+
+############################################################################
+# Set up for update
+############################################################################
+
+mkdir $settingsdir unless (-d $settingsdir);
+
+# Connect to the system log
+
+openlog( "ipblacklist", "nofatal", LOG_USER);
+log_message LOG_INFO, "Starting IP Blacklist processing";
+
+# Read settings
+
+General::readhash( $settings, \%settings ) if (-e $settings);
+General::readhash( $checked, \%checked ) if (-e $checked);
+General::readhash( $modified, \%modified ) if (-e $modified);
+General::readhash( $proxy_settings, \%proxy_settings ) if (-e $proxy_settings);
+
+if (-r $sources)
+{
+ debug 1, "Reading sources file";
+
+ eval qx|/bin/cat $sources|;
+}
+
+# Find out the red interface name
+
+if (-r $red_setting)
+{
+ open IN, '<', $red_setting or die "Can't open red interface name file: $!";
+
+ $red_iface = <IN>;
+ chomp $red_iface;
+
+ close IN;
+}
+
+if (@ARGV)
+{
+ foreach my $cmd (@ARGV)
+ {
+ if ('update' =~ m/^$cmd/i)
+ {
+ # Called hourly when enabled and on setting changes.
+ # Update the blacklists.
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ do_update();
+ }
+ }
+ elsif ('start' =~ m/^$cmd/i)
+ {
+ # Called during system startup.
+ # Restore saved blacklists.
+ # Don't do an update since that takes too long.
+
+ do_start() if ($settings{'ENABLE'} eq 'on');
+ }
+ elsif ('stop' =~ m/^$cmd/i)
+ {
+ # Called when shutting down.
+ # Delete IPSets and IPTables chains
+
+ do_stop();
+ }
+ elsif ('restore' =~ m/^$cmd/i)
+ {
+ # Called after restoring backup.
+ # Delete IPSets and IPTables chains, then re-create them with the new
+ # (restored) settings and make sure updates are enabled.
+
+ do_stop();
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ do_start();
+ enable_updates();
+ }
+ else
+ {
+ disable_updates();
+ }
+ }
+ elsif ('log-on' =~ m/^$cmd/i)
+ {
+ # Called from WUI.
+ # Create entries in IPTables chains to log dropped packets.
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ enable_logging();
+ }
+ }
+ elsif ('log-off' =~ m/^$cmd/i)
+ {
+ # Called from WUI
+ # Delete entries in IPTables chains to log dropped packets.
+
+ disable_logging();
+ }
+ elsif ('enable' =~ m/^$cmd/i)
+ {
+ # Called from WUI to enable blacklists
+ # Do an update and then enable automatic updates
+
+ if ($settings{'ENABLE'} eq 'on')
+ {
+ do_update();
+ enable_updates();
+ }
+ }
+ elsif ('disable' =~ m/^$cmd/i)
+ {
+ # Called from WUI to disable blacklists.
+ # Disable updates, delete IPSets, IPTables chains and save files.
+
+ disable_updates();
+ do_delete();
+ }
+ elsif ('autoblacklist-update' =~ m/^$cmd/i)
+ {
+ # Updates AUTOBLACKLIST options
+
+ autoblacklist_update();
+ }
+ elsif ('autoblacklist-clear' =~ m/^$cmd/i)
+ {
+ # Clears AUTOBLACKLIST contents
+
+ autoblacklist_clear();
+ }
+ else
+ {
+ print "Usage: $0 [update|start|stop|restart|log-on|log-off|enable|disable|autoblacklist-update|autoblacklist-clear]\n";
+ }
+ }
+}
+elsif ($settings{'ENABLE'} eq 'on')
+{
+ do_update();
+}
+
+stop_ipset();
+
+if ($update_status)
+{
+ debug 1, "Writing updated status file";
+
+ General::writehash( $checked, \%checked );
+ General::writehash( $modified, \%modified );
+}
+
+# Remove the pid file
+
+unlink $lockfile;
+
+log_message LOG_INFO, "Finished IP Blacklist processing";
+closelog();
+
+
+#------------------------------------------------------------------------------
+# sub do_stop
+#
+# Deletes all the IPTables chains and the IPSets
+#------------------------------------------------------------------------------
+
+sub do_stop()
+{
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Stopping IP Blacklists";
+
+ foreach my $list ( $autoblacklist, sort keys %sources )
+ {
+ if (exists $chains{$list})
+ {
+ delete_list( $list );
+ }
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_start
+#
+# Recreates the IPTables chains and the IPSets from the saved values
+#------------------------------------------------------------------------------
+
+sub do_start()
+{
+ log_message LOG_NOTICE, "Starting IP Blacklists";
+
+ foreach my $list ( sort keys %sources )
+ {
+ if (-e "$savedir/$list.conf")
+ {
+ log_message LOG_INFO, "Restoring blacklist $list";
+ system( "$ipset restore -f $savedir/$list.conf" );
+
+ create_list( $list );
+ }
+ }
+
+ if ($settings{$autoblacklist} eq 'on')
+ {
+ create_autoblacklist();
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_delete
+#
+# Deletes the IPTables chains, the IPSets and the saved values.
+#------------------------------------------------------------------------------
+
+sub do_delete()
+{
+ # Get the list of current ipsets
+
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Deleting IP Blacklists";
+
+ foreach my $source ( sort keys %sources )
+ {
+ if (exists $chains{$source})
+ {
+ delete_list( $source );
+ }
+
+ if (-e "$savedir/$source.conf")
+ {
+ unlink "$savedir/$source.conf";
+ }
+ }
+
+ if ($settings{$autoblacklist} eq 'on')
+ {
+ delete_autoblacklist();
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub do_update
+#
+# Updates all the blacklists.
+# Creates or deletes the blacklist firewall rules as necessary and checks for
+# updates to the blacklists.
+#------------------------------------------------------------------------------
+
+sub do_update()
+{
+ return unless (is_connected());
+
+ my $type = 'hash:ip';
+
+ # Get the list of current ipsets
+
+ get_ipsets();
+
+ # Check sources
+
+ debug 1, "Checking blacklist sources";
+
+ foreach my $list ( sort keys %sources )
+ {
+ my @new_blacklist = ();
+ my $name = $sources{$list}{'name'};
+ my $rate = $sources{$list}{'rate'};
+ my $last_checked = $checked{$list} || 0;
+ my $enabled = 0;
+
+ if (exists $modified{$list})
+ {
+ # Limit the check rate to the minimum defined in the WUI, unless we're
+ # creating the list
+
+ $rate = $settings{'RATE'} if ($settings{'RATE'} > $rate);
+ }
+
+ if (exists $settings{$list})
+ {
+ $enabled = $settings{$list} eq 'on';
+ }
+
+ debug 1, "Checking blacklist source: $name";
+
+ if ($enabled)
+ {
+ # Has enough time passed since the last time we checked the list?
+
+ if (($last_checked + $rate * $hours) < (time() + $margin))
+ {
+ download_list( $list, \@new_blacklist, \$type );
+
+ next unless (@new_blacklist);
+
+ if (not exists $chains{$list})
+ {
+ # Doesn't currently exist: Create it.
+
+ create_ipset( $list, $type, scalar @new_blacklist );
+ create_list( $list );
+ }
+
+ update_list( $list, \@new_blacklist, $type );
+ }
+ }
+ elsif (exists $chains{$list})
+ {
+ # Exists, but not enabled: Delete it.
+
+ delete_list( $list );
+
+ # Delete the save file
+ # Don't delete the checked time from the status, in case the list is
+ # re-enabled quickly - don't want to exceed maximum allowed download
+ # rate.
+
+ unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+ delete $modified{$list} if (exists $modified{$list});
+ $update_status = 1;
+ }
+ }
+
+ # Check for any deleted lists
+
+ foreach my $list (keys %sources)
+ {
+ if (not exists $sources{$list})
+ {
+ delete_list( $list );
+
+ # Delete the save file
+
+ unlink "$savedir/$list.conf" if (-e "$savedir/$list.conf");
+
+ # Delete from the status
+
+ delete $modified{$list} if (exists $modified{$list});
+ delete $checked{$list} if (exists $checked{$list});
+ $update_status = 1;
+ }
+ }
+
+ if ($settings{$autoblacklist} eq 'on')
+ {
+ create_autoblacklist() if (not exists $chains{$autoblacklist});
+ }
+ else
+ {
+ delete_autoblacklist() if (exists $chains{$autoblacklist});
+ }
+
+ log_message LOG_INFO, "Completed IP Blacklist update";
+}
+
+
+#------------------------------------------------------------------------------
+# sub autoblacklist_update()
+#
+# Updates the settings for the AUTOBLACKLIST
+#------------------------------------------------------------------------------
+
+sub autoblacklist_update()
+{
+ # Get the list of current ipsets
+
+ get_ipsets();
+
+ # Delete the existing AUTOBLACKLIST, if it currently exists.
+
+ delete_autoblacklist() if (exists $chains{$autoblacklist});
+
+ # Re-create the AUTOBLACKLIST with the correct parameters.
+
+ create_autoblacklist() if ($settings{$autoblacklist} eq 'on');
+}
+
+
+#------------------------------------------------------------------------------
+# sub autoblacklist_clear()
+#
+# Clears the contents of the AUTOBLACKLIST
+#------------------------------------------------------------------------------
+
+sub autoblacklist_clear()
+{
+ log_message LOG_INFO, "Flush Automatic blacklist";
+ ipset( "flush $autoblacklist" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub is_connected()
+#
+# Checks that the system is connected to the internet.
+#
+# This looks for a file created by IPFire when connected to the internet
+#------------------------------------------------------------------------------
+
+sub is_connected()
+{
+ return (-e "${General::swroot}/red/active");
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_list( list )
+#
+# Creates a new IPTables chain for a blacklist source.
+# The set must be created before calling this function.
+#
+# Parameters:
+# list The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub create_list( $ )
+{
+ my ($list) = @_;
+
+ log_message LOG_INFO, "Create IPTables chains for blacklist $list";
+
+ # Create new chain in filter table
+
+ iptables( " -N ${list}_BLOCK" ) == 0 or
+ ( abort "Could not create IPTables chain ${list}_BLOCK", return );
+
+ # Add the logging and drop rules
+
+ if ($settings{'LOGGING'} eq 'on')
+ {
+ iptables( "-A ${list}_BLOCK -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" ) == 0 or
+ ( abort "Could not create IPTables chain $list LOG rule", return );
+ }
+
+ iptables( "-A ${list}_BLOCK -j DROP" ) == 0 or
+ ( abort "Could not create IPTables chain $list drop rule", return );
+
+ # Add the rules to check against the set
+
+ iptables( "-A IPBLACKLISTREDIN -p ALL -m set --match-set $list src -j ${list}_BLOCK" );
+ iptables( "-A IPBLACKLISTREDOUT -p ALL -m set --match-set $list dst -j ${list}_BLOCK" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_autoblacklist()
+#
+# Creates a new IPTables chain for the AUTOBLACKLIST. This also creates the
+# IPSet with the correct timeout.
+#------------------------------------------------------------------------------
+
+sub create_autoblacklist()
+{
+ return unless ($red_iface); # Can't add rule to policy unless this is set
+
+ # Create the set for the AUTOBLACKLIST
+
+ ipset( "create $autoblacklist hash:ip timeout $settings{BLOCK_PERIOD}" );
+
+ # Create new chain in filter table
+
+ create_list( $autoblacklist );
+
+ # For the AUTOBLACKLIST there are extra rules to reset the timeout on the
+ # blockled addresses
+
+ iptables( "-I ${autoblacklist}_BLOCK -m set --match-set $autoblacklist src -j SET --add-set $autoblacklist src --exist" );
+ iptables( "-I ${autoblacklist}_BLOCK -m set --match-set $autoblacklist dst -j SET --add-set $autoblacklist dst --exist" );
+
+ # For the AUTOBLACKLIST there is an extra rule to add an entry to the list
+ # of blocked addresses. This is added to the input policy chain.
+
+ iptables( "-I POLICYIN 1 -i $red_iface -m hashlimit --hashlimit-mode srcip --hashlimit-above $settings{BLOCK_THRESHOLD}/hour --hashlimit-name $autoblacklist -j SET --add-set $autoblacklist src" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub delete_list( $list )
+#
+# Deletes an IPTables chain when a blacklist source is disabled. Also flushes
+# and destroys the IPSet.
+#
+# Parameters:
+# list The name of the blacklist
+#------------------------------------------------------------------------------
+
+sub delete_list( $ )
+{
+ my ($list) = @_;
+
+ log_message LOG_INFO, "Delete IPTables chains for blacklist $list";
+
+ # Remove the blacklist chains from the main INPUT and OUTPUT chains
+
+ iptables( "-D IPBLACKLISTREDIN -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not remove IPSet $list from IPBLACKLISTREDIN chain";
+
+ iptables( "-D IPBLACKLISTREDOUT -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not remove IPSet $list from IPBLACKLISTREDOUT chain";
+
+ # Flush and delete the chain
+
+ iptables( "-F ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not flush IPTables chain ${list}_BLOCK";
+
+ iptables( "-X ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not delete IPTables chain ${list}_BLOCK";
+
+ # Flush and delete the set
+
+ ipset( "flush $list" );
+
+ ipset( "destroy $list" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub delete_autoblacklist()
+#
+# Deletes the autoblacklist IPTables chain when it is disabled. Also flushes
+# and destroys the IPSet.
+#------------------------------------------------------------------------------
+
+sub delete_autoblacklist()
+{
+ # For the AUTOBLACKLIST there is an extra rule to remove
+
+ unless ($red_iface)
+ {
+ iptables( "-D POLICYIN -i $red_iface -m hashlimit --hashlimit-mode srcip --hashlimit-above $settings{BLOCK_THRESHOLD}/hour --hashlimit-name $autoblacklist -j SET --add-set $autoblacklist src" );
+ }
+
+ # Now do a normal delete
+
+ delete_list( $autoblacklist );
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_list( chain, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist. Depending on the blacklist one of
+# two methods are used:
+#
+# - For some lists the header is downloaded and the modification date checked.
+# If newer than the existing list, the update is downloaded.
+# - For other lists this is not supported,so the whole file has to be
+# downloaded regardless.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+# list The name of the blacklist
+# ref_list A reference to an array to store the downloaded blacklist
+# ref_type A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_list( $$$ )
+{
+ my ($list, $new_blacklist, $type) = @_;
+
+ $checked{$list} = time();
+ $update_status = 1;
+
+ # Check the parser for the blacklist
+
+ if (not exists $parsers{ $sources{$list}{'parser'} })
+ {
+ log_message LOG_ERR, "Can't find parser $sources{$list}{'parser'} for $list blacklist";
+ return;
+ }
+
+ if ($sources{$list}{'method'} eq 'check-header-time')
+ {
+ download_check_header_time( $list, $new_blacklist, $type );
+ }
+ else
+ {
+ download_wget( $list, $new_blacklist, $type );
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_check_header_time( chain, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist. The header is downloaded and the
+# modification date checked. If newer than the existing list, the update is
+# downloaded.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+# list The name of the blacklist
+# ref_list A reference to an array to store the downloaded blacklist
+# ref_type A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_check_header_time( $$$ )
+{
+ my ($list, $new_blacklist, $type) = @_;
+ my $found_ip = 0;
+ my $found_net = 0;
+
+ # Get the parser for the blacklist
+
+ my $parser = $parsers{ $sources{$list}{'parser'} };
+
+ log_message LOG_INFO, "Checking modification time for blacklist $list update with LWP";
+
+ # Create a user agent for downloading the blacklist
+ # Limit the download size for safety (10 MiB)
+
+ my $ua = LWP::UserAgent->new( max_size => 10485760 );
+
+ # Get the Proxy settings
+
+ if ($proxy_settings{'UPSTREAM_PROXY'})
+ {
+ if ($proxy_settings{'UPSTREAM_USER'})
+ {
+ $ua->proxy("http" => "http://$proxy_settings{'UPSTREAM_USER'}:$proxy_settings{'UPSTREAM_PASSWORD'}\@$proxy_settings{'UPSTREAM_PROXY'}/");
+ $ua->proxy("https" => "http://$proxy_settings{'UPSTREAM_USER'}:$proxy_settings{'UPSTREAM_PASSWORD'}\@$proxy_settings{'UPSTREAM_PROXY'}/");
+ }
+ else
+ {
+ $ua->proxy("http" => "http://$proxy_settings{'UPSTREAM_PROXY'}/");
+ $ua->proxy("https" => "http://$proxy_settings{'UPSTREAM_PROXY'}/");
+ }
+ }
+
+ # Get the blacklist modification time from the internet
+
+ my $request = HTTP::Request->new( HEAD => $sources{$list}{'url'} );
+
+ my $response = $ua->request( $request );
+
+ if (not $response->is_success)
+ {
+ log_message LOG_WARNING, "Failed to download $list header $sources{$list}{'url'}: ". $response->status_line;
+
+ return;
+ }
+
+ # Has the blacklist been modified since we last read it?
+
+ if (exists $modified{$list} and $modified{$list} >= $response->last_modified)
+ {
+ # We've already got this version of the blacklist
+
+ debug 1, "Blacklist $list not modified";
+ return;
+ }
+
+ debug 1, "Blacklist $list Modification times: old " . $modified{$list} . ", new " . $response->last_modified if (exists $modified{$list});
+ log_message LOG_INFO, "Downloading blacklist $list with LWP";
+
+ # Download the blacklist
+
+ $request = HTTP::Request->new( GET => $sources{$list}{'url'} );
+ $response = $ua->request($request);
+
+ if (not $response->is_success)
+ {
+ log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: ". $response->status_line;
+
+ return;
+ }
+
+ $modified{$list} = $response->last_modified;
+
+ foreach my $line (split /[\r\n]+/, $response->content)
+ {
+ chomp $line;
+
+ my $address = &$parser( $line );
+
+ next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);
+
+ if ($address =~ m|/\d+|)
+ {
+ $found_net = 1;
+ }
+ else
+ {
+ $found_ip = 1;
+ }
+
+ push @{ $new_blacklist }, $address;
+ }
+
+ if ($found_net and $found_ip)
+ {
+ # Convert mixed address and network set to all network
+
+ foreach my $address (@{ $new_blacklist })
+ {
+ $address .= '/32' unless ($address =~ m|/\d+|);
+ }
+
+ $found_ip = 0;
+ }
+
+ $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub download_wget( chain, ref_list, ref_type )
+#
+# Updates the IP Addresses for a blacklist. The whole file is download with
+# wget and then the modification time compared with the stored modification
+# time. If the update is newer then the downloaded list is parsed.
+#
+# Once downloaded the list is parsed to get the IP addresses and/or networks.
+#
+# Parameters:
+# list The name of the blacklist
+# ref_list A reference to an array to store the downloaded blacklist
+# ref_type A reference to store the type of the blacklist
+#------------------------------------------------------------------------------
+
+sub download_wget( $$$ )
+{
+ my ($list, $new_blacklist, $type) = @_;
+ my $wget_proxy = '';
+ my $found_ip = 0;
+ my $found_net = 0;
+
+ my $parser = $parsers{ $sources{$list}{'parser'} };
+
+ log_message LOG_INFO, "Downloading blacklist $list update with wget";
+
+ # Get the Proxy settings
+
+ if ($proxy_settings{'UPSTREAM_PROXY'})
+ {
+ if ($proxy_settings{'UPSTREAM_USER'})
+ {
+ $wget_proxy = "--proxy=on --proxy-user=$proxy_settings{'UPSTREAM_USER'} --proxy-passwd=$proxy_settings{'UPSTREAM_PASSWORD'} -e http_proxy=http://$proxy_settings{'UPSTREAM_PROXY'}/";
+ }
+ else
+ {
+ $wget_proxy = "--proxy=on -e http_proxy=http://$proxy_settings{'UPSTREAM_PROXY'}/";
+ }
+ }
+
+ my $retv = system( "wget $wget_proxy --no-show-progress -o $detailed_log -O $tmpdir/ipblacklist_$list $sources{$list}{'url'}" );
+
+ if ($retv != 0)
+ {
+ my $error = $wget_status[ $retv/256 ];
+ log_message LOG_WARNING, "Failed to download $list blacklist $sources{$list}{'url'}: $error";
+ return;
+ }
+
+ my @file_info = stat( "$tmpdir/ipblacklist_$list" );
+
+ if (exists $modified{$list} and $modified{$list} >= $file_info[9])
+ {
+ # We've already got this version of the blocklist
+
+ debug 1, "Blacklist $list not modified";
+ unlink "$tmpdir/ipblacklist_$list";
+ return;
+ }
+
+ open LIST, '<', "$tmpdir/ipblacklist_$list" or (abort "Can't open downloaded blacklist for $list: $!", return);
+
+ $modified{$list} = $file_info[9];
+
+ foreach my $line (<LIST>)
+ {
+ chomp $line;
+
+ my $address = &$parser( $line );
+
+ next unless ($address);
+ next unless ($address =~ m|\d+\.\d+\.\d+\.\d+|);
+
+ if ($address =~ m|/\d+|)
+ {
+ $found_net = 1;
+ }
+ else
+ {
+ $found_ip = 1;
+ }
+
+ push @{ $new_blacklist }, $address;
+ }
+
+ close LIST;
+
+ unlink "$tmpdir/ipblacklist_$list";
+
+ if ($found_net and $found_ip)
+ {
+ # Convert mixed address and network set to all network
+
+ foreach my $address (@{ $new_blacklist })
+ {
+ $address .= '/32' unless ($address =~ m|/\d+|);
+ }
+
+ $found_ip = 0;
+ }
+
+ $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub read_ipset( list, old, type )
+#
+# Reads the existing contents of the set
+#
+# Parameters:
+# chain The name of the blacklist
+# old Reference to array to contain blacklist
+# type Reference to type
+#------------------------------------------------------------------------------
+
+sub read_ipset( $$$ )
+{
+ my ($list, $old, $type) = @_;
+ my $found_net = 0;
+ my $found_ip = 0;
+
+ debug 2, "Reading existing ipset for blacklist $list";
+
+ foreach my $line (qx/$ipset list $list/)
+ {
+ next unless ($line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|);
+
+ my $address = $1;
+
+ if (($address =~ m|/\d+$|) and ($address !~ m|/32$|))
+ {
+ $found_net = 1;
+ }
+ else
+ {
+ $found_ip = 1;
+ $address =~ s|/32$||;
+ }
+
+ $$old{$address} = 1;
+ }
+
+ if ($found_ip and $found_net)
+ {
+ # Convert mixed address and network set to all network
+
+ my @ads_list = keys %{ $old };
+
+ foreach my $address (@ads_list)
+ {
+ unless ($address =~ m|/\d+|)
+ {
+ delete $$old{$address};
+ $$old{"$address/32"} = 1;
+ }
+ }
+
+ $found_ip = 0;
+ }
+
+ $$type = $found_net ? 'hash:net' : 'hash:ip';
+}
+
+
+#------------------------------------------------------------------------------
+# sub update_list( chain, new, new_type )
+#
+# Updates the IP Addresses for a blacklist
+#
+# The new list is compared to the existing list and new entries added or old
+# entries deleted as necessary.
+#
+# Parameters:
+# list The name of the blacklist
+# new Reference to array of new blacklist entries
+# new_type The type of the updated list (hash:ip or hash:net)
+#------------------------------------------------------------------------------
+
+sub update_list( $$$ )
+{
+ my ($list, $new, $new_type) = @_;
+ my %old;
+ my $old_type;
+ my $changes = 0;
+
+ debug 2, "Checking for $list blacklist update from $sources{$list}{'url'}";
+
+ log_message LOG_INFO, "Updating $list blacklist";
+
+ read_ipset( $list, \%old, \$old_type );
+
+ # Check the IPSet type hasn't changed
+
+ if ($new_type ne $old_type)
+ {
+ # Change the IPSet type. This requires removing references to it first.
+ # We could delete and then create the chain, but doing it like this keeps
+ # the statistics.
+
+ log_message LOG_NOTICE, "Blacklist $list changed type from $old_type to $new_type";
+
+ # Remove the IPSet from the IPTables chains
+
+ iptables( "-D 'IPBLACKLISTREDIN' -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not remove ${list} from IPBLACKLISTREDIN chain";
+
+ iptables( "-D 'IPBLACKLISTREDOUT' -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not remove ${list} from IPBLACKLISTREDOUT chain";
+
+ # Flush and delete the old set
+
+ ipset( "flush $list" );
+ ipset( "destroy $list" );
+
+ %old = ();
+
+ # Create the new ipset
+
+ create_ipset( $list, $new_type, scalar @{ $new } );
+
+ # Add the rules to check against the set
+
+ iptables( "-A 'IPBLACKLISTREDIN' -p ALL -m set --match-set $list src -j ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not add IPSet $list to IPBLACKLISTREDIN chain";
+
+ iptables( "-A 'IPBLACKLISTREDOUT' -p ALL -m set --match-set $list dst -j ${list}_BLOCK" ) == 0 or
+ log_message LOG_ERR, "Could not add IPSet $list to IPBLACKLISTREDOUT chain";
+ }
+
+ # Process the blacklist
+
+ foreach my $address ( @{ $new } )
+ {
+ # We've got an address. Add to IPSet if it's new
+
+ if (exists $old{$address})
+ {
+ delete $old{$address}; # Not new
+ }
+ else
+ {
+ ipset( "add $list $address -exist" );
+
+ $changes++;
+ }
+
+ debug 3, "Add net $address to blacklist $list";
+ }
+
+ # Delete old entries that aren't needed any more
+
+ debug 2, "Removing deleted rules from IPTables chain for blacklist $list";
+
+ foreach my $address ( keys %old )
+ {
+ ipset( "del $list $address" );
+
+ $changes++;
+
+ debug 3, "Delete old net $address from blacklist $list";
+ }
+
+
+ log_message LOG_INFO, "Finished updating $list blacklist with $changes changes";
+
+ # Save the blacklist for the next reboot
+
+ mkdir "$savedir" unless (-d "$savedir" );
+
+ ipset( "save $list -file $savedir/$list.conf" );
+
+ stop_ipset();
+}
+
+
+#------------------------------------------------------------------------------
+# sub get_ipsets( )
+#
+# Gets a list of the current IPSets
+#------------------------------------------------------------------------------
+
+sub get_ipsets( )
+{
+ debug 1, "Reading list of existing ipsets";
+
+ my @sets = qx($ipset -n list);
+
+ # Parse the tables
+
+ foreach my $line (@sets)
+ {
+ chomp $line;
+
+ next unless ($line);
+
+ $chains{$line} = 1;
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub create_ipset( name, type, size )
+#
+# Creates a new IPSet. The current and maximum size of the set are determined
+# by taking the next power of two greater than the numer of entries, subject to
+# a minimum size. This allows for future expansion.
+#
+# Parameters:
+# name The name of the blacklist
+# type The type of the blacklist (hash:ip or hash:net)
+# size The number of entries in the lsit
+#------------------------------------------------------------------------------
+
+sub create_ipset( $$$ )
+{
+ my ($name, $type, $size) = @_;
+
+ my $hashsize = 1;
+ $hashsize <<= 1 while ($hashsize < $size);
+ my $maxsize = ($hashsize < 16384) ? 32768 : $hashsize * 2;
+
+ # Create the new ipset
+ ipset( "create $name $type hashsize $hashsize maxelem $maxsize" );
+ stop_ipset(); # Need to do this to action the IPSet commands
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_logging()
+#
+# Enable logging of packets dropped by IP Blacklist rules.
+#------------------------------------------------------------------------------
+
+sub enable_logging()
+{
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Enabling IP Blacklist logging";
+
+ foreach my $list ( sort keys %sources )
+ {
+ if (exists $chains{$list})
+ {
+ iptables( "-I ${list}_BLOCK 1 -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" );
+ }
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_logging()
+#
+# Disable logging of packets dropped by IP Blacklist rules.
+#------------------------------------------------------------------------------
+
+sub disable_logging()
+{
+ get_ipsets();
+
+ log_message LOG_NOTICE, "Disabling IP Blacklist logging";
+
+ foreach my $list ( sort keys %sources )
+ {
+ if (exists $chains{$list})
+ {
+ iptables( "-D ${list}_BLOCK -j LOG -m limit --limit 10/second --log-prefix 'DROP_$list'" );
+ }
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub enable_updates()
+#
+# Adds a command to the fcrontab to run the update hourly.
+# The update is executed at an offset from the hour so that all the users don't
+# try to download the updates at exactly the same time - the blacklists are
+# provided free, so it's good manners to spread the load on the servers.
+#------------------------------------------------------------------------------
+
+sub enable_updates()
+{
+ my @lines = qx/$fcrontab -l/;
+ my $found = 0;
+
+ # Check for an existing fcrontab entry.
+
+ foreach my $line (@lines)
+ {
+ if ($line =~ m|/usr/local/bin/ipblacklist|)
+ {
+ return if ($line !~ m/^#/); # Already enabled
+
+ # Found - uncomment the line
+
+ $line =~ s/^#+//;
+ $found = 1;
+ log_message LOG_INFO, "Enable IP Address Blacklist update in crontab";
+ last;
+ }
+ }
+
+ if (not $found)
+ {
+ # Add a new entry
+
+ my $start = int( rand(50) ) + 5;
+
+ push @lines, "\n";
+ push @lines, "# IP Blacklist update\n";
+ push @lines, "\%hourly,nice(1),random,serial $start /usr/local/bin/ipblacklist\n";
+ log_message LOG_INFO, "Add IP Address Blacklist update to crontab";
+ }
+
+ open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+ print FCRONTAB @lines;
+ close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub disable_updates()
+#
+# Comments out the entry in the fcrontab that runs the updates.
+#------------------------------------------------------------------------------
+
+sub disable_updates()
+{
+ my @lines = qx/$fcrontab -l/;
+ my $found = 0;
+
+ foreach my $line (@lines)
+ {
+ if ($line =~ m|/usr/local/bin/ipblacklist|)
+ {
+ return if ($line =~ m/^#/); # Already disabled
+
+ # Found - comment the line
+
+ $line =~ s/^#*/#/;
+ $found = 1;
+ log_message LOG_INFO, "Disable IP Address Blacklist updates";
+ last;
+ }
+ }
+
+ return unless ($found); # Don't update crontab unnecessarily
+
+ open FCRONTAB, "| $fcrontab -" or (abort "Can't open pipe to write fcrontab: $!", return);
+ print FCRONTAB @lines;
+ close FCRONTAB;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_text_with_hash_comments( line )
+#
+# Parses an input line removing comments.
+#
+# Parameters:
+# line The line to parse
+#
+# Returns:
+# Either an IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_text_with_hash_comments( $ )
+{
+ my ($line) = @_;
+
+ return "" if ($line =~ m/^\s*#/);
+
+ $line =~ s/#.*$//;
+
+ $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;
+
+ return $1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_text_with_semicolon_comments( line )
+#
+# Parses an input line removing comments.
+#
+# Parameters:
+# line The line to parse
+#
+# Returns:
+# Either and IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_text_with_semicolon_comments( $ )
+{
+ my ($line) = @_;
+
+ return "" if ($line =~ m/^\s*;/);
+
+ $line =~ s/;.*$//;
+
+ $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;
+
+ return $1;
+}
+
+
+#------------------------------------------------------------------------------
+# sub parse_dshield( line )
+#
+# Parses an input line removing comments.
+#
+# The format is:
+# Start Addrs End Addrs Netmask Nb Attacks Network Name Country email
+# We're only interested in the start address and netmask.
+#
+# Parameters:
+# line The line to parse
+#
+# Returns:
+# Either and IP Address or a null string
+#------------------------------------------------------------------------------
+
+sub parse_dshield( $ )
+{
+ my ($line) = @_;
+
+ return "" if ($line =~ m/^\s*#/);
+
+ $line =~ s/#.*$//;
+
+ $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)|;
+
+ return unless ($1);
+ return "$1/32" unless ($2);
+
+ return "$1/$2";
+}
+
+
+#------------------------------------------------------------------------------
+# sub iptables( cmd )
+#
+# Executes an IPTables command, waiting for the lock to ensure only one change
+# is made at a time.
+#
+# Parameters:
+# cmd The command to execute
+#
+# Returns:
+# Status of command
+#------------------------------------------------------------------------------
+
+sub iptables( $ )
+{
+ my ($cmd) = @_;
+
+ return system( "$iptables $cmd" );
+}
+
+
+#------------------------------------------------------------------------------
+# sub ipset( cmd )
+#
+# Executes an IPSet command. The command is piped to a sub-process running
+# ipset, rather than exected separately. This saves the overhead of starting a
+# new process for each command. The sub-process is started if it's not already
+# running.
+#
+# Parameters:
+# cmd The command to execute
+#------------------------------------------------------------------------------
+
+sub ipset( $ )
+{
+ my ($cmd) = @_;
+
+ if (not $ipset_running)
+ {
+ local $SIG{PIPE} = 'IGNORE';
+ open IPSET, "|-", $ipset, "restore" or die "Can't start ipset: $!";
+ $ipset_running = 1;
+ }
+
+ print IPSET "$cmd\n";
+}
+
+
+#------------------------------------------------------------------------------
+# sub stop_ipset( )
+#
+# Stops the ipset sub-process.
+#------------------------------------------------------------------------------
+
+sub stop_ipset( )
+{
+ if ($ipset_running)
+ {
+ close IPSET or abort "ipset process died: $! $?";
+ $ipset_running = 0;
+ }
+}
+
+
+#------------------------------------------------------------------------------
+# sub abort( message, parameters... )
+#
+# Aborts the current activity, printing out an error message.
+#
+# Parameters:
+# message Message to be printed
+#------------------------------------------------------------------------------
+
+sub abort( $ )
+{
+ my ($message) = @_;
+
+ log_message( LOG_ERR, $message );
+ carp $message;
+}
+
+
+#------------------------------------------------------------------------------
+# sub log_message( level, message )
+#
+# Logs a message. If the script is run from a terminal messages are also
+# output on STDOUT.
+#
+# Parameters:
+# level Severity of message
+# message Message to be logged
+#------------------------------------------------------------------------------
+
+sub log_message( $$ )
+{
+ my ($level, $message) = @_;
+
+ print "($level) $message\n" if (-t STDIN);
+ syslog( $level, $message );
+}
+
+
+#------------------------------------------------------------------------------
+# sub debug( level, message )
+#
+# Optionally logs a debug message. If the script is run from a terminal, level
+# 1 debug messages are output regardless of the debug setting.
+#
+# Parameters:
+# level Debug level
+# message Message to be logged
+#------------------------------------------------------------------------------
+
+sub debug( $$ )
+{
+ my ($level, $message) = @_;
+
+ if (($level <= $settings{'DEBUG'}) or
+ ($level == 1 and -t STDIN))
+ {
+ log_message LOG_DEBUG, $message;
+ }
+}