From patchwork Sat Apr 6 04:29:35 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Tim FitzGeorge X-Patchwork-Id: 2178 Return-Path: Received: from mail01.ipfire.org (mail01.i.ipfire.org [172.28.1.200]) (using TLSv1.2 with cipher ECDHE-ECDSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mail01.ipfire.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by web07.i.ipfire.org (Postfix) with ESMTPS id 7F21E861F37 for ; Fri, 5 Apr 2019 18:30:50 +0100 (BST) Received: from mail01.i.ipfire.org (localhost [IPv6:::1]) by mail01.ipfire.org (Postfix) with ESMTP id 44bRfx6jMjz5Lcxl; Fri, 5 Apr 2019 18:30:49 +0100 (BST) Received: from smtp.hosts.co.uk (smtp.hosts.co.uk [85.233.160.19]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mail01.ipfire.org (Postfix) with ESMTPS id 44bRfh16hqz5D0P7 for ; Fri, 5 Apr 2019 18:30:36 +0100 (BST) Received: from [31.127.205.161] (helo=aragorn.tfitzgeorge.me.uk) by smtp.hosts.co.uk with esmtpa (Exim) (envelope-from ) id 1hCSfi-00006W-4B; Fri, 05 Apr 2019 18:30:35 +0100 From: Tim FitzGeorge To: development@lists.ipfire.org Subject: [PATCH 07/12] statusmail: Plugins for services Date: Fri, 5 Apr 2019 18:29:35 +0100 Message-Id: <20190405172940.13168-8-ipfr@tfitzgeorge.me.uk> X-Mailer: git-send-email 2.16.4 In-Reply-To: <20190405172940.13168-1-ipfr@tfitzgeorge.me.uk> References: <20190405172940.13168-1-ipfr@tfitzgeorge.me.uk> X-Spamd-Result: default: False [-13.17 / 11.00]; ARC_NA(0.00)[]; RCVD_VIA_SMTP_AUTH(0.00)[]; RECEIVED_SPAMHAUS_PBL(0.00)[161.205.127.31.zen.spamhaus.org : 127.0.0.11]; FROM_HAS_DN(0.00)[]; TO_DN_SOME(0.00)[]; R_SPF_ALLOW(-0.20)[+ip4:85.233.160.19]; MIME_GOOD(-0.10)[text/plain]; RCVD_TLS_LAST(0.00)[]; REPLY(-4.00)[]; DMARC_NA(0.00)[tfitzgeorge.me.uk]; TO_MATCH_ENVRCPT_SOME(0.00)[]; MX_GOOD(-0.01)[cached: mx1.ukservers.net]; RCPT_COUNT_TWO(0.00)[2]; MID_CONTAINS_FROM(1.00)[]; NEURAL_HAM(-2.99)[-0.996,0]; IP_SCORE(-3.77)[ip: (-9.91), ipnet: 85.233.160.0/19(-4.96), asn: 8622(-3.96), country: GB(-0.04)]; RCVD_IN_DNSWL_LOW(-0.10)[19.160.233.85.list.dnswl.org : 127.0.5.1]; R_DKIM_NA(0.00)[]; MIME_TRACE(0.00)[0:+]; ASN(0.00)[asn:8622, ipnet:85.233.160.0/19, country:GB]; RCVD_COUNT_TWO(0.00)[2]; BAYES_HAM(-3.00)[100.00%]; FROM_EQ_ENVFROM(0.00)[] Authentication-Results: mail01.ipfire.org; dkim=none; dmarc=none; spf=pass (mail01.ipfire.org: domain of ipfr@tfitzgeorge.me.uk designates 85.233.160.19 as permitted sender) smtp.mailfrom=ipfr@tfitzgeorge.me.uk X-BeenThere: development@lists.ipfire.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: IPFire development talk List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: development-bounces@lists.ipfire.org Sender: "Development" Intrusion Prevention System plugin works with Suricata, but not Snort Signed-off-by: Tim FitzGeorge --- .../services_intrusion_prevention_system.pm | 239 ++++++++++++++++++ src/statusmail/plugins/services_urlfilter.pm | 275 +++++++++++++++++++++ 2 files changed, 514 insertions(+) create mode 100644 src/statusmail/plugins/services_intrusion_prevention_system.pm create mode 100644 src/statusmail/plugins/services_urlfilter.pm diff --git a/src/statusmail/plugins/services_intrusion_prevention_system.pm b/src/statusmail/plugins/services_intrusion_prevention_system.pm new file mode 100644 index 000000000..4ca174d4e --- /dev/null +++ b/src/statusmail/plugins/services_intrusion_prevention_system.pm @@ -0,0 +1,239 @@ +#!/usr/bin/perl + +############################################################################ +# # +# Send log and status emails for IPFire # +# # +# This is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 3 of the License, or # +# (at your option) any later version. # +# # +# This is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with IPFire; if not, write to the Free Software # +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +# # +# Copyright (C) 2018 - 2019 The IPFire Team # +# # +############################################################################ + +use strict; +use warnings; + +require "${General::swroot}/lang.pl"; + +package Services_Intrusion_Prevention_System; + +use Time::Local; + +############################################################################ +# Function prototypes +############################################################################ + +sub get_log( $ ); + +############################################################################ +# Constants +############################################################################ + +use constant { SEC => 0, + MIN => 1, + HOUR => 2, + MDAY => 3, + MON => 4, + YEAR => 5, + WDAY => 6, + YDAY => 7, + ISDST => 8, + MONSTR => 9 }; + +############################################################################ +# BEGIN Block +# +# Register the log items available in this file +############################################################################ + +sub BEGIN +{ + main::add_mail_item( 'ident' => 'services-ips-alerts', + 'section' => $Lang::tr{'services'}, + 'subsection' => $Lang::tr{'intrusion prevention system'}, + 'item' => $Lang::tr{'statusmail ips alerts'}, + 'function' => \&alerts, + 'option' => { 'type' => 'integer', + 'name' => $Lang::tr{'statusmail ips min priority'}, + 'min' => 1, + 'max' => 4 } ); +} + +############################################################################ +# Code +############################################################################ + +#------------------------------------------------------------------------------ +# sub get_log +# +# +#------------------------------------------------------------------------------ + +sub get_log( $ ) +{ + my ($this) = @_; + +# There's only one data item, so don't use the cache +# my $data = $this->cache( 'ips-alerts' ); +# return $data if (defined $data); + + my $name = '/var/log/suricata/fast.log'; + + my %info; + my $last_mon = 0; + my $last_day = 0; + my $last_hour = 0; + my $last_time = 0; + my $time = 0; + my $now = time(); + my $year = 0; + my $start_time = $this->get_period_start; + my $end_time = $this->get_period_end; + my @stats; + + for (my $filenum = $this->get_number_weeks ; $filenum >= 0 ; $filenum--) + { + my $filename = $filenum < 1 ? $name : "$name.$filenum"; + + if (-r "$filename.gz") + { + @stats = stat( _ ); + next if ($stats[9] < $start_time); + + open IN, "gzip -dc $filename.gz |" or next; + } + elsif (-r $filename) + { + @stats = stat( _ ); + open IN, '<', $filename or next; + } + else + { + next; + } + + foreach my $line () + { + chomp $line; + + # Alerts have the format: + # + # mm/dd/yyyy-hh:mm:ss.uuuuuu [Action] [**] [gid:sid:prio] message [**] [Classification: type] [Priority: prio] {protocol} src-ip:src-port -> dest-ip:dest-port + + $line =~ s/^\s+//; + $line =~ s/\s+$//; + + next unless ($line); + + my ($mon, $day, $year, $hour, $min, $sec, $gid, $sid, $message, $prio, $src, $dest) = + $line =~ m|(\d+)/(\d+)/(\d+)-(\d+):(\d+):(\d+)\.\d+\s+\[\w+\]\s+\[\*\*\]\s+\[(\d+):(\d+):\d+\]\s*(.*)\s+\[\*\*\].*\[Priority:\s(\d+)\].*?\s+(\d+\.\d+\.\d+\.\d+(?::\d+)?) -> (\d+\.\d+\.\d+\.\d+(?::\d+)?)|; + + $sid = "$gid-$sid"; + + if ($mon != $last_mon or $day != $last_day or $hour != $last_hour) + { + # Hour, day or month changed. Convert to unix time so we can work out + # whether the message time falls between the limits we're interested in. + + my @time; + + $time[YEAR] = $year; + + ($time[MON], $time[MDAY], $time[HOUR], $time[MIN], $time[SEC]) = ($mon - 1, $day, $hour, $min, $sec); + + $time = timelocal( @time ); + + ($last_mon, $last_day, $last_hour) = ($mon, $day, $hour); + } + + # Check to see if we're within the specified limits. + # Note that the minutes and seconds may be incorrect, but since we only deal + # in hour boundaries this doesn't matter. + + next if ($time < $start_time); + last if ($time > $end_time); + + my $timestr = "$mon/$day $hour:$min:$sec"; + + $info{total}++; + + if (exists $info{by_sid}{$sid}) + { + $info{by_sid}{$sid}{count}++; + $info{by_sid}{$sid}{last} = $timestr; + } + else + { + $info{by_sid}{$sid}{count} = 1; + $info{by_sid}{$sid}{priority} = $prio; + $info{by_sid}{$sid}{message} = $message; + $info{by_sid}{$sid}{first} = $timestr; + $info{by_sid}{$sid}{last} = $timestr; + } + } + + close IN; + } + +# $this->cache( 'ids-alerts', \%info ); + + return \%info; + +} + + +#------------------------------------------------------------------------------ + +sub alerts( $$ ) +{ + my ($self, $min_priority) = @_; + my @table; + + use Sort::Naturally; + + push @table, ['|', '|', '<', '|', '|', '|', '|']; + push @table, [ 'SID', $Lang::tr{'priority'}, $Lang::tr{'name'}, $Lang::tr{'count'}, $Lang::tr{'percentage'}, $Lang::tr{'first'}, $Lang::tr{'last'} ]; + + my $stats = get_log( $self ); + + foreach my $sid (sort { $$stats{by_sid}{$a}{priority} <=> $$stats{by_sid}{$b}{priority} || + $$stats{by_sid}{$b}{count} <=> $$stats{by_sid}{$a}{count}} keys %{ $$stats{by_sid} } ) + { + my $message = $$stats{by_sid}{$sid}{message}; + my $priority = $$stats{by_sid}{$sid}{priority}; + my $count = $$stats{by_sid}{$sid}{count}; + my $first = $$stats{by_sid}{$sid}{first}; + my $last = $$stats{by_sid}{$sid}{last}; + my $percent = int( 100 * $count / $$stats{total} + 0.5); + + last if ($priority > $min_priority); + + $message = $self->split_string( $message, 40 ); + + push @table, [ $sid, $priority, $message, $count, $percent, $first, $last ]; + } + + if (@table > 2) + { + $self->add_table( @table ); + + return 1; + } + + return 0; +} + + +1; diff --git a/src/statusmail/plugins/services_urlfilter.pm b/src/statusmail/plugins/services_urlfilter.pm new file mode 100644 index 000000000..620dc1e20 --- /dev/null +++ b/src/statusmail/plugins/services_urlfilter.pm @@ -0,0 +1,275 @@ +#!/usr/bin/perl + +############################################################################ +# # +# Send log and status emails for IPFire # +# # +# This is free software; you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation; either version 3 of the License, or # +# (at your option) any later version. # +# # +# This is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with IPFire; if not, write to the Free Software # +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +# # +# Copyright (C) 2018 - 2019 The IPFire Team # +# # +############################################################################ + +require "${General::swroot}/lang.pl"; + +use strict; +use warnings; + +package Services_Urlfilter; + +use Time::Local; + +############################################################################ +# BEGIN Block +# +# Register the log items available in this file +############################################################################ + +sub BEGIN +{ + main::add_mail_item( 'ident' => 'services-urlfilter-client', + 'section' => $Lang::tr{'services'}, + 'subsection' => $Lang::tr{'urlfilter url filter'}, + 'item' => $Lang::tr{'urlfilter client'}, + 'function' => \&clients, + 'option' => { 'type' => 'integer', + 'name' => $Lang::tr{'statusmail urlfilter min count'}, + 'min' => 1, + 'max' => 1000 } ); + + main::add_mail_item( 'ident' => 'services-urlfilter-destination', + 'section' => $Lang::tr{'services'}, + 'subsection' => $Lang::tr{'urlfilter url filter'}, + 'item' => $Lang::tr{'destination'}, + 'function' => \&destinations, + 'option' => { 'type' => 'integer', + 'name' => $Lang::tr{'statusmail urlfilter min count'}, + 'min' => 1, + 'max' => 1000 } ); +} + +############################################################################ +# Constants +############################################################################ + +use constant { SEC => 0, + MIN => 1, + HOUR => 2, + MDAY => 3, + MON => 4, + YEAR => 5, + WDAY => 6, + YDAY => 7, + ISDST => 8, + MONSTR => 9 }; + + +############################################################################ +# Functions +############################################################################ + +sub get_log( $ ); + +#------------------------------------------------------------------------------ +# sub get_log( this ) +# +# Gets messages from the log files that relate to the URL filter. The data is +# cached sot hat a second call does not process the logs again. +# +# Parameters: +# this message object +#------------------------------------------------------------------------------ + +sub get_log( $ ) +{ + my ($this) = @_; + + my $data = $this->cache( 'urlfilter' ); + return $data if (defined $data); + + my %info; + my $weeks = $this->get_number_weeks; + my @start_time = $this->get_period_start;; + my @end_time = $this->get_period_end; + + # Iterate over the log files + + foreach my $name (glob '/var/log/squidGuard/*\.log') + { + next if ($name =~ m/squidGuard.log/); + + # Iterate over old versions of the file + + for (my $filenum = $weeks ; $filenum >= 0 ; $filenum--) + { + my $filename = $filenum < 1 ? $name : "$name.$filenum"; + + if (-r "$filename.gz") + { + open IN, "gzip -dc $filename.gz |" or next; + } + elsif (-r $filename) + { + open IN, '<', $filename or next; + } + else + { + next; + } + + # Scan the file + + foreach my $line () + { + my ($year, $mon, $day, $hour) = split /[\s:-]+/, $line; + + # Check to see if we're within the specified limits. + # Note that the minutes and seconds may be incorrect, but since we only deal + # in hour boundaries this doesn't matter. + + next if (($year < ($start_time[YEAR]+1900)) or + ($year == ($start_time[YEAR]+1900) and $mon < ($start_time[MON]+1)) or + ($year == ($start_time[YEAR]+1900) and $mon == ($start_time[MON]+1) and $day < $start_time[MDAY]) or + ($year == ($start_time[YEAR]+1900) and $mon == ($start_time[MON]+1) and $day == $start_time[MDAY] and $hour < $start_time[HOUR])); + + last if (($year > ($end_time[YEAR]+1900)) or + ($year == ($end_time[YEAR]+1900) and $mon > ($end_time[MON]+1)) or + ($year == ($end_time[YEAR]+1900) and $mon == ($end_time[MON]+1) and $day > $end_time[MDAY]) or + ($year == ($end_time[YEAR]+1900) and $mon == ($end_time[MON]+1) and $day == $end_time[MDAY] and $hour > $end_time[HOUR])); + + # Is it an entry we're interested in? + + next unless ($line =~ m/Request/); + + # Process the entry + + if (my ($date, $time, $pid, $type, $destination, $client) = split / /, $line) + { + $destination =~ s#^http://|^https://##; + $destination =~ s/\/.*$//; + $destination =~ s/:\d+$//; + my $site = substr( $destination, 0, 69 ); + $site .= "..." if (length( $destination ) > 69); + + my @category = split /\//, $type; + + my ($address, $name) = split "/", $client; + + $this->set_host_name( $address, $name ) unless ($address eq $name); + + $info{'client'}{$address}++; + $info{'destination'}{"$site||$category[1]"}++; + $info{'count'}++; + } + } + + close IN; + } + } + + $this->cache( 'urlfilter', \%info ); + + return \%info; +} + + +#------------------------------------------------------------------------------ +# sub clients( this, min_count ) +# +# Output information on the systems trying to access forbidden destinations. +# +# Parameters: +# this message object +# min_count don't output information on clients accessing less than this +# number of destinations +#------------------------------------------------------------------------------ + +sub clients( $$ ) +{ + my ($self, $min_count) = @_; + my @table; + + use Sort::Naturally; + + push @table, [ $Lang::tr{'urlfilter client'}, $Lang::tr{'count'} ]; + + my $stats = get_log( $self ); + + foreach my $client (sort { $$stats{'client'}{$b} <=> $$stats{'client'}{$a} } keys %{ $$stats{'client'} } ) + { + my $count = $$stats{'client'}{$client}; + last if ($count < $min_count); + + my $host = $self->lookup_ip_address( $client ); + + $client .= "\n$host" if ($host); + + push @table, [ $client, $count ]; + } + + if (@table > 1) + { + $self->add_table( @table ); + + return 1; + } + + return 0; +} + + +#------------------------------------------------------------------------------ +# sub destinations( this, min_count ) +# +# Output information on the forbidden destinations being accessed. +# +# Parameters: +# this message object +# min_count don't output information on destinations accessed less than this +# number of times. +#------------------------------------------------------------------------------ + +sub destinations( $$ ) +{ + my ($self, $min_count) = @_; + my @table; + + use Sort::Naturally; + + push @table, [ $Lang::tr{'destination'}, $Lang::tr{'urlfilter category'}, $Lang::tr{'count'} ]; + + my $stats = get_log( $self ); + + foreach my $key (sort { $$stats{'destination'}{$b} <=> $$stats{'destination'}{$a} } keys %{ $$stats{'destination'} } ) + { + my $count = $$stats{'destination'}{$key}; + last if ($count < $min_count); + + my ($destination, $category) = split /\|\|/, $key; + + push @table, [ $destination, $category, $count ]; + } + + if (@table > 1) + { + $self->add_table( @table ); + + return 1; + } + + return 0; +} + +1;