From de1199e2a32e9f177ea237392b0ae22b5f8a2b87 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 21 Mar 2022 19:52:04 +0100 Subject: [PATCH 01/77] ids-functions.pl: Drop downloader code for sourcefire based ruleset. Even if the servers do not support HEAD requests, the remote filesize (content_length) can be obtained from the connection headers. This generic method works for all servers and therefore we do not need the code for handle sourcefire servers in a different way anymore. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 43 +++++---------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 94dccc8ae..eb276030b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -354,43 +354,6 @@ sub downloadruleset ($) { return 1; } - # Variable to store the filesize of the remote object. - my $remote_filesize; - - # The sourcfire (snort rules) does not allow to send "HEAD" requests, so skip this check - # for this webserver. - # - # Check if the ruleset source contains "snort.org". - unless ($url =~ /\.snort\.org/) { - # Pass the requrested url to the downloader. - my $request = HTTP::Request->new(HEAD => $url); - - # Accept the html header. - $request->header('Accept' => 'text/html'); - - # Perform the request and fetch the html header. - my $response = $downloader->request($request); - - # Check if there was any error. - unless ($response->is_success) { - # Obtain error. - my $error = $response->status_line(); - - # Log error message. - &_log_to_syslog("Unable to download the ruleset. \($error\)"); - - # Return "1" - false. - return 1; - } - - # Assign the fetched header object. - my $header = $response->headers(); - - # Grab the remote file size from the object and store it in the - # variable. - $remote_filesize = $header->content_length; - } - # Load perl module to deal with temporary files. use File::Temp; @@ -416,6 +379,12 @@ sub downloadruleset ($) { return 1; } + # Obtain the connection headers. + my $headers = $response->headers; + + # Get the remote size of the downloaded file. + my $remote_filesize = $headers->content_length; + # Load perl stat module. use File::stat; From e71804fb821acf84ef4ad06fcaf80dda6fe8af0c Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 21 Mar 2022 20:19:25 +0100 Subject: [PATCH 02/77] ids-functions.pl: Allow "3" download attempts for each provider before fail. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 38 ++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index eb276030b..29d186995 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -256,6 +256,10 @@ sub downloadruleset ($) { # If no provider is given default to "all". $provider //= 'all'; + # The amount of download attempts before giving up and + # logging an error. + my $max_dl_attempts = 3; + # Hash to store the providers and access id's, for which rules should be downloaded. my %sheduled_providers = (); @@ -364,19 +368,33 @@ sub downloadruleset ($) { # Pass the requested url to the downloader. my $request = HTTP::Request->new(GET => $url); - # Perform the request and save the output into the tmpfile. - my $response = $downloader->request($request, $tmpfile); + my $dl_attempt = 1; + my $response; - # Check if there was any error. - unless ($response->is_success) { - # Obtain error. - my $error = $response->content; + # Download and retry on failure. + while ($dl_attempt <= $max_dl_attempts) { + # Perform the request and save the output into the tmpfile. + $response = $downloader->request($request, $tmpfile); - # Log error message. - &_log_to_syslog("Unable to download the ruleset. \($error\)"); + # Check if the download was successfull. + if($response->is_success) { + # Break loop. + last; - # Return "1" - false. - return 1; + # Check if we ran out of download re-tries. + } elsif ($dl_attempt eq $max_dl_attempts) { + # Obtain error. + my $error = $response->content; + + # Log error message. + &_log_to_syslog("Unable to download the ruleset. \($error\)"); + + # Return "1" - false. + return 1; + } + + # Increase download attempt counter. + $dl_attempt++; } # Obtain the connection headers. From 304ce130fd1e19de6a4faf9834784e0d821c02c1 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 21 Mar 2022 20:21:21 +0100 Subject: [PATCH 03/77] ids-functions.pl: Remove temporary file, if the download failed. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 29d186995..90b7d2d7f 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -393,6 +393,9 @@ sub downloadruleset ($) { return 1; } + # Remove temporary file, if one exists. + unlink("$tmpfile") if (-e "$tmpfile"); + # Increase download attempt counter. $dl_attempt++; } From 7d8956083b76babafef3c8e82fb32c4f243424c3 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 24 Mar 2022 20:18:58 +0100 Subject: [PATCH 04/77] ids-functions.pl: Early load required perl modules. This will help us to determine if all required perl modules and their dependencies are avail and load-able. It also prevents us from doubble loading modules and makes development and maintainance more easy. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 66 ++++++++++++++------------------- 1 file changed, 27 insertions(+), 39 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 90b7d2d7f..35b8f7a83 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -29,6 +29,33 @@ require '/var/ipfire/general-functions.pl'; require "${General::swroot}/network-functions.pl"; require "${General::swroot}/suricata/ruleset-sources"; +# Load perl module to deal with Archives. +use Archive::Tar; + +# Load perl module to deal with files and path. +use File::Basename; + +# Load module to move files. +use File::Copy; + +# Load module to recursely remove files and a folder. +use File::Path qw(rmtree); + +# Load module to get file stats. +use File::stat; + +# Load module to deal with temporary files. +use File::Temp; + +# Load the libwwwperl User Agent module. +use LWP::UserAgent; + +# Load function from posix module to format time strings. +use POSIX qw (strftime); + +# Load module to talk to the kernel log daemon. +use Sys::Syslog qw(:DEFAULT setlogsock); + # Location where all config and settings files are stored. our $settingsdir = "${General::swroot}/suricata"; @@ -280,9 +307,6 @@ sub downloadruleset ($) { my %proxysettings=(); &General::readhash("${General::swroot}/proxy/settings", \%proxysettings); - # Load required perl module to handle the download. - use LWP::UserAgent; - # Init the download module. # # Request SSL hostname verification and specify path @@ -358,9 +382,6 @@ sub downloadruleset ($) { return 1; } - # Load perl module to deal with temporary files. - use File::Temp; - # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); my $tmpfile = $tmp->filename(); @@ -406,9 +427,6 @@ sub downloadruleset ($) { # Get the remote size of the downloaded file. my $remote_filesize = $headers->content_length; - # Load perl stat module. - use File::stat; - # Perform stat on the tmpfile. my $stat = stat($tmpfile); @@ -443,9 +461,6 @@ sub downloadruleset ($) { return 1; } - # Load file copy module, which contains the move() function. - use File::Copy; - # Overwrite the may existing rulefile or tarball with the downloaded one. move("$tmpfile", "$dl_rulesfile"); @@ -469,18 +484,9 @@ sub downloadruleset ($) { sub extractruleset ($) { my ($provider) = @_; - # Load perl module to deal with archives. - use Archive::Tar; - # Disable chown functionality when uncompressing files. $Archive::Tar::CHOWN = "0"; - # Load perl module to deal with files and path. - use File::Basename; - - # Load perl module for file copying. - use File::Copy; - # Get full path and downloaded rulesfile for the given provider. my $tarball = &_get_dl_rulesfile($provider); @@ -586,9 +592,6 @@ sub extractruleset ($) { # Extract the file to the temporary directory. $tar->extract_file("$packed_file", "$destination"); } else { - # Load perl module to deal with temporary files. - use File::Temp; - # Generate temporary file name, located in the temporary rules directory and a suffix of ".tmp". my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "$tmp_rules_directory", UNLINK => 0 ); my $tmpfile = $tmp->filename(); @@ -637,9 +640,6 @@ sub oinkmaster () { &extractruleset($provider); } - # Load perl module to talk to the kernel syslog. - use Sys::Syslog qw(:DEFAULT setlogsock); - # Establish the connection to the syslog service. openlog('oinkmaster', 'cons,pid', 'user'); @@ -811,9 +811,6 @@ sub merge_sid_msg (@) { ## the rules directory. # sub move_tmp_ruleset() { - # Load perl module. - use File::Copy; - # Do a directory listing of the temporary directory. opendir DH, $tmp_rules_directory; @@ -831,8 +828,6 @@ sub move_tmp_ruleset() { ## Function to cleanup the temporary IDS directroy. # sub cleanup_tmp_directory () { - # Load rmtree() function from file path perl module. - use File::Path 'rmtree'; # Delete temporary directory and all containing files. rmtree([ "$tmp_directory" ]); @@ -860,9 +855,6 @@ sub log_error ($) { sub _log_to_syslog ($) { my ($message) = @_; - # Load perl module to talk to the kernel syslog. - use Sys::Syslog qw(:DEFAULT setlogsock); - # The syslog function works best with an array based input, # so generate one before passing the message details to syslog. my @syslog = ("ERR", " $message"); @@ -1584,10 +1576,6 @@ sub get_ruleset_date($) { my $date; my $mtime; - # Load neccessary perl modules for file stat and to format the timestamp. - use File::stat; - use POSIX qw( strftime ); - # Get the stored rulesfile for this provider. my $stored_rulesfile = &_get_dl_rulesfile($provider); From f264adda359ec58846840e60d9743ca522fa4004 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 24 Mar 2022 20:29:21 +0100 Subject: [PATCH 05/77] ids-functions.pl: Re-order download request handler creation. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 35b8f7a83..c0e64a3a2 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -382,13 +382,13 @@ sub downloadruleset ($) { return 1; } + # Pass the requested URL to the downloader. + my $request = HTTP::Request->new(GET => $url); + # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); my $tmpfile = $tmp->filename(); - # Pass the requested url to the downloader. - my $request = HTTP::Request->new(GET => $url); - my $dl_attempt = 1; my $response; From 061391e77601082727e64e40dfa352f89be18ce1 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 24 Mar 2022 20:51:56 +0100 Subject: [PATCH 06/77] ids-functions.pl: Use If-Modified-Since header to reduce file downloads. When using the "If-Modified-Since" header, the server can be requested if a modified version of the file can be served. In case that is true, the file will be sent and stored by the downloader function. If the file has not been touched since the last time, the server will respond with the code "304" (Not modified). This tells us, that the current stored file is the latest one (still up-to-date) and we safely can skip the download attempt for this provider. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 48 ++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index c0e64a3a2..61aecc250 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -47,6 +47,9 @@ use File::stat; # Load module to deal with temporary files. use File::Temp; +# Load module to deal with the date formats used by the HTTP protocol. +use HTTP::Date; + # Load the libwwwperl User Agent module. use LWP::UserAgent; @@ -386,9 +389,32 @@ sub downloadruleset ($) { my $request = HTTP::Request->new(GET => $url); # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". + # The downloaded file will be stored there until some sanity checks are performed. my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); my $tmpfile = $tmp->filename(); + # Call function to get the final path and filename for the downloaded file. + my $dl_rulesfile = &_get_dl_rulesfile($provider); + + # Check if the rulesfile already exits, because it has been downloaded in the past. + # + # In this case we are requesting the server if the remote file has been changed or not. + # This will be done by sending the modification time in a special HTTP header. + if (-f $dl_rulesfile) { + # Call stat on the file. + my $stat = stat($dl_rulesfile); + + # Omit the mtime of the existing file. + my $mtime = $stat->mtime; + + # Convert the timestamp into right format. + my $http_date = time2str($mtime); + + # Add the If-Modified-Since header to the request to ask the server if the + # file has been modified. + $request->header( 'If-Modified-Since' => "$http_date" ); + } + my $dl_attempt = 1; my $response; @@ -402,6 +428,14 @@ sub downloadruleset ($) { # Break loop. last; + # Check if the server responds with 304 (Not Modified). + } elsif ($response->code == 304) { + # Log to syslog. + &_log_to_syslog("Ruleset is up-to-date, no update required."); + + # Nothing to do, the ruleset is up-to-date. + return; + # Check if we ran out of download re-tries. } elsif ($dl_attempt eq $max_dl_attempts) { # Obtain error. @@ -424,6 +458,10 @@ sub downloadruleset ($) { # Obtain the connection headers. my $headers = $response->headers; + # Get the timestamp from header, when the file has been modified the + # last time. + my $last_modified = $headers->last_modified; + # Get the remote size of the downloaded file. my $remote_filesize = $headers->content_length; @@ -446,9 +484,6 @@ sub downloadruleset ($) { return 1; } - # Genarate and assign file name and path to store the downloaded rules file. - my $dl_rulesfile = &_get_dl_rulesfile($provider); - # Check if a file name could be obtained. unless ($dl_rulesfile) { # Log error message. @@ -464,6 +499,13 @@ sub downloadruleset ($) { # Overwrite the may existing rulefile or tarball with the downloaded one. move("$tmpfile", "$dl_rulesfile"); + # Check if we got a last-modified value from the server. + if ($last_modified) { + # Assign the last-modified timestamp as mtime to the + # rules file. + utime(time(), "$last_modified", "$dl_rulesfile"); + } + # Delete temporary file. unlink("$tmpfile"); From 235e3e92a32a95339c177a94371b22c4bc0877a6 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 24 Mar 2022 21:17:08 +0100 Subject: [PATCH 07/77] ids-functions.pl: Add get_subscription_code() function. This function can be used to obtain the subscription code of a given configured provider. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 61aecc250..6819d7879 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1607,6 +1607,34 @@ END close(FILE); } +# +## Function to get the subscription code of a configured provider. +# +sub get_subscription_code($) { + my ($provider) = @_; + + my %configured_providers = (); + + # Read-in providers settings file. + &General::readhasharray($providers_settings_file, \%configured_providers); + + # Loop through the hash of configured providers. + foreach my $id (keys %configured_providers) { + # Assign nice human-readable values to the data fields. + my $provider_handle = $configured_providers{$id}[0]; + my $subscription_code = $configured_providers{$id}[1]; + + # Check if the current processed provider is the requested one. + if ($provider_handle eq $provider) { + # Return the obtained subscription code. + return $subscription_code; + } + } + + # No subscription code found - return nothing. + return; +} + # ## Function to get the ruleset date for a given provider. ## From 0f2c5211f6d8b183a8496ff208c20ca5ddc0c6c6 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 24 Mar 2022 21:17:59 +0100 Subject: [PATCH 08/77] ids-functions.pl: Limit downloader to only one provider. Remove the option and required code to download the rulesets for all configured and enabled providers by just calling the downloader function. This cause a lot of troubles and if required, directly should be handled by the processing script. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 285 ++++++++++++++------------------ 1 file changed, 121 insertions(+), 164 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 6819d7879..56ee94541 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -269,43 +269,21 @@ sub checkdiskspace () { } # -## This function is responsible for downloading the configured IDS rulesets or if no one is specified -## all configured rulesets will be downloaded. +## This function is responsible for downloading the ruleset for a given provider. ## -## * At first it gathers all configured ruleset providers, initialize the downloader and sets an -## upstream proxy if configured. -## * After that, the given ruleset or in case all rulesets should be downloaded, it will determine wether it -## is enabled or not. +## * At first it initialize the downloader and sets an upstream proxy if configured. ## * The next step will be to generate the final download url, by obtaining the URL for the desired -## ruleset, add the settings for the upstream proxy. -## * Finally the function will grab all the rules files or tarballs from the servers. +## ruleset and add the settings for the upstream proxy. +## * Finally the function will grab the rule file or tarball from the server. +## It tries to reduce the amount of download by using the "If-Modified-Since" HTTP header. # sub downloadruleset ($) { my ($provider) = @_; - # If no provider is given default to "all". - $provider //= 'all'; - # The amount of download attempts before giving up and # logging an error. my $max_dl_attempts = 3; - # Hash to store the providers and access id's, for which rules should be downloaded. - my %sheduled_providers = (); - - # Get used provider settings. - my %used_providers = (); - &General::readhasharray("$providers_settings_file", \%used_providers); - - # Check if a ruleset has been configured. - unless(%used_providers) { - # Log that no ruleset has been configured and abort. - &_log_to_syslog("No ruleset provider has been configured."); - - # Return "1". - return 1; - } - # Read proxysettings. my %proxysettings=(); &General::readhash("${General::swroot}/proxy/settings", \%proxysettings); @@ -342,177 +320,156 @@ sub downloadruleset ($) { $downloader->proxy(['http', 'https'], $proxy_url); } - # Loop through the hash of configured providers. - foreach my $id ( keys %used_providers ) { - # Skip providers which are not enabled. - next if ($used_providers{$id}[3] ne "enabled"); + # Log download/update of the ruleset. + &_log_to_syslog("Downloading ruleset for provider: $provider."); - # Obtain the provider handle. - my $provider_handle = $used_providers{$id}[0]; + # Grab the download url for the provider. + my $url = $IDS::Ruleset::Providers{$provider}{'dl_url'}; + + # Check if the provider requires a subscription. + if ($IDS::Ruleset::Providers{$provider}{'requires_subscription'} eq "True") { + # Grab the subscription code. + my $subscription_code = &get_subscription_code($provider); + + # Add the subscription code to the download url. + $url =~ s/\/$subscription_code/g; - # Handle update off all providers. - if (($provider eq "all") || ($provider_handle eq "$provider")) { - # Add provider handle and it's id to the hash of sheduled providers. - $sheduled_providers{$provider_handle} = $id; - } } - # Loop through the hash of sheduled providers. - foreach my $provider ( keys %sheduled_providers) { - # Log download/update of the ruleset. - &_log_to_syslog("Downloading ruleset for provider: $provider."); + # Abort if no url could be determined for the provider. + unless ($url) { + # Log error and abort. + &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider."); + return 1; + } - # Grab the download url for the provider. - my $url = $IDS::Ruleset::Providers{$provider}{'dl_url'}; + # Pass the requested URL to the downloader. + my $request = HTTP::Request->new(GET => $url); - # Check if the provider requires a subscription. - if ($IDS::Ruleset::Providers{$provider}{'requires_subscription'} eq "True") { - # Grab the previously stored access id for the provider from hash. - my $id = $sheduled_providers{$provider}; + # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". + # The downloaded file will be stored there until some sanity checks are performed. + my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); + my $tmpfile = $tmp->filename(); - # Grab the subscription code. - my $subscription_code = $used_providers{$id}[1]; + # Call function to get the final path and filename for the downloaded file. + my $dl_rulesfile = &_get_dl_rulesfile($provider); - # Add the subscription code to the download url. - $url =~ s/\/$subscription_code/g; + # Check if the rulesfile already exits, because it has been downloaded in the past. + # + # In this case we are requesting the server if the remote file has been changed or not. + # This will be done by sending the modification time in a special HTTP header. + if (-f $dl_rulesfile) { + # Call stat on the file. + my $stat = stat($dl_rulesfile); - } + # Omit the mtime of the existing file. + my $mtime = $stat->mtime; - # Abort if no url could be determined for the provider. - unless ($url) { - # Log error and abort. - &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider."); - return 1; - } + # Convert the timestamp into right format. + my $http_date = time2str($mtime); - # Pass the requested URL to the downloader. - my $request = HTTP::Request->new(GET => $url); + # Add the If-Modified-Since header to the request to ask the server if the + # file has been modified. + $request->header( 'If-Modified-Since' => "$http_date" ); + } - # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". - # The downloaded file will be stored there until some sanity checks are performed. - my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); - my $tmpfile = $tmp->filename(); + my $dl_attempt = 1; + my $response; - # Call function to get the final path and filename for the downloaded file. - my $dl_rulesfile = &_get_dl_rulesfile($provider); + # Download and retry on failure. + while ($dl_attempt <= $max_dl_attempts) { + # Perform the request and save the output into the tmpfile. + $response = $downloader->request($request, $tmpfile); - # Check if the rulesfile already exits, because it has been downloaded in the past. - # - # In this case we are requesting the server if the remote file has been changed or not. - # This will be done by sending the modification time in a special HTTP header. - if (-f $dl_rulesfile) { - # Call stat on the file. - my $stat = stat($dl_rulesfile); + # Check if the download was successfull. + if($response->is_success) { + # Break loop. + last; - # Omit the mtime of the existing file. - my $mtime = $stat->mtime; + # Check if the server responds with 304 (Not Modified). + } elsif ($response->code == 304) { + # Log to syslog. + &_log_to_syslog("Ruleset is up-to-date, no update required."); - # Convert the timestamp into right format. - my $http_date = time2str($mtime); + # Nothing to do, the ruleset is up-to-date. + return; - # Add the If-Modified-Since header to the request to ask the server if the - # file has been modified. - $request->header( 'If-Modified-Since' => "$http_date" ); - } + # Check if we ran out of download re-tries. + } elsif ($dl_attempt eq $max_dl_attempts) { + # Obtain error. + my $error = $response->content; - my $dl_attempt = 1; - my $response; - - # Download and retry on failure. - while ($dl_attempt <= $max_dl_attempts) { - # Perform the request and save the output into the tmpfile. - $response = $downloader->request($request, $tmpfile); - - # Check if the download was successfull. - if($response->is_success) { - # Break loop. - last; - - # Check if the server responds with 304 (Not Modified). - } elsif ($response->code == 304) { - # Log to syslog. - &_log_to_syslog("Ruleset is up-to-date, no update required."); - - # Nothing to do, the ruleset is up-to-date. - return; - - # Check if we ran out of download re-tries. - } elsif ($dl_attempt eq $max_dl_attempts) { - # Obtain error. - my $error = $response->content; - - # Log error message. - &_log_to_syslog("Unable to download the ruleset. \($error\)"); - - # Return "1" - false. - return 1; - } - - # Remove temporary file, if one exists. - unlink("$tmpfile") if (-e "$tmpfile"); - - # Increase download attempt counter. - $dl_attempt++; - } - - # Obtain the connection headers. - my $headers = $response->headers; - - # Get the timestamp from header, when the file has been modified the - # last time. - my $last_modified = $headers->last_modified; - - # Get the remote size of the downloaded file. - my $remote_filesize = $headers->content_length; - - # Perform stat on the tmpfile. - my $stat = stat($tmpfile); - - # Grab the local filesize of the downloaded tarball. - my $local_filesize = $stat->size; - - # Check if both file sizes match. - if (($remote_filesize) && ($remote_filesize ne $local_filesize)) { # Log error message. - &_log_to_syslog("Unable to completely download the ruleset. "); - &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. "); - - # Delete temporary file. - unlink("$tmpfile"); + &_log_to_syslog("Unable to download the ruleset. \($error\)"); # Return "1" - false. return 1; } - # Check if a file name could be obtained. - unless ($dl_rulesfile) { - # Log error message. - &_log_to_syslog("Unable to store the downloaded rules file. "); + # Remove temporary file, if one exists. + unlink("$tmpfile") if (-e "$tmpfile"); - # Delete downloaded temporary file. - unlink("$tmpfile"); + # Increase download attempt counter. + $dl_attempt++; + } - # Return "1" - false. - return 1; - } + # Obtain the connection headers. + my $headers = $response->headers; - # Overwrite the may existing rulefile or tarball with the downloaded one. - move("$tmpfile", "$dl_rulesfile"); + # Get the timestamp from header, when the file has been modified the + # last time. + my $last_modified = $headers->last_modified; - # Check if we got a last-modified value from the server. - if ($last_modified) { - # Assign the last-modified timestamp as mtime to the - # rules file. - utime(time(), "$last_modified", "$dl_rulesfile"); - } + # Get the remote size of the downloaded file. + my $remote_filesize = $headers->content_length; + + # Perform stat on the tmpfile. + my $stat = stat($tmpfile); + + # Grab the local filesize of the downloaded tarball. + my $local_filesize = $stat->size; + + # Check if both file sizes match. + if (($remote_filesize) && ($remote_filesize ne $local_filesize)) { + # Log error message. + &_log_to_syslog("Unable to completely download the ruleset. "); + &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. "); # Delete temporary file. unlink("$tmpfile"); - # Set correct ownership for the tarball. - set_ownership("$dl_rulesfile"); + # Return "1" - false. + return 1; } + # Check if a file name could be obtained. + unless ($dl_rulesfile) { + # Log error message. + &_log_to_syslog("Unable to store the downloaded rules file. "); + + # Delete downloaded temporary file. + unlink("$tmpfile"); + + # Return "1" - false. + return 1; + } + + # Overwrite the may existing rulefile or tarball with the downloaded one. + move("$tmpfile", "$dl_rulesfile"); + + # Check if we got a last-modified value from the server. + if ($last_modified) { + # Assign the last-modified timestamp as mtime to the + # rules file. + utime(time(), "$last_modified", "$dl_rulesfile"); + } + + # Delete temporary file. + unlink("$tmpfile"); + + # Set correct ownership for the tarball. + set_ownership("$dl_rulesfile"); + # If we got here, everything worked fine. Return nothing. return; } From e26edcc1c7cac6235f7d60c527f980895fc3fe5a Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 25 Mar 2022 06:03:40 +0100 Subject: [PATCH 09/77] ids-functions.pl: Provide better return codes, if the downloader fails. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 56ee94541..5b299dc44 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -277,6 +277,13 @@ sub checkdiskspace () { ## * Finally the function will grab the rule file or tarball from the server. ## It tries to reduce the amount of download by using the "If-Modified-Since" HTTP header. # +## Return codes: +## +## * "no url" - If no download URL could be gathered for the provider. +## * "not modified" - In case the already stored rules file is up to date. +## * "incomplete download" - When the remote file size differs from the downloaded file size. +## * "$error" - The error message generated from the LWP::User Agent module. +# sub downloadruleset ($) { my ($provider) = @_; @@ -340,7 +347,7 @@ sub downloadruleset ($) { unless ($url) { # Log error and abort. &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider."); - return 1; + return "no url"; } # Pass the requested URL to the downloader. @@ -391,8 +398,8 @@ sub downloadruleset ($) { # Log to syslog. &_log_to_syslog("Ruleset is up-to-date, no update required."); - # Nothing to do, the ruleset is up-to-date. - return; + # Return "not modified". + return "not modified"; # Check if we ran out of download re-tries. } elsif ($dl_attempt eq $max_dl_attempts) { @@ -402,8 +409,8 @@ sub downloadruleset ($) { # Log error message. &_log_to_syslog("Unable to download the ruleset. \($error\)"); - # Return "1" - false. - return 1; + # Return the error message from response.. + return "$error"; } # Remove temporary file, if one exists. @@ -439,7 +446,7 @@ sub downloadruleset ($) { unlink("$tmpfile"); # Return "1" - false. - return 1; + return "incomplete download"; } # Check if a file name could be obtained. From 52a557a848c3f744278aec91d7e16ff1f5c24833 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:14:40 +0100 Subject: [PATCH 10/77] ids-functions.pl: Remove logging calls from downloader. The download script should not directly do the logging stuff. It simply should download the files for the requested provider and return an error code on fail. The logging should be done at another place. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 5b299dc44..f4541ce77 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -327,9 +327,6 @@ sub downloadruleset ($) { $downloader->proxy(['http', 'https'], $proxy_url); } - # Log download/update of the ruleset. - &_log_to_syslog("Downloading ruleset for provider: $provider."); - # Grab the download url for the provider. my $url = $IDS::Ruleset::Providers{$provider}{'dl_url'}; @@ -343,12 +340,8 @@ sub downloadruleset ($) { } - # Abort if no url could be determined for the provider. - unless ($url) { - # Log error and abort. - &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider."); - return "no url"; - } + # Abort and return "no url", if no url could be determined for the provider. + return "no url" unless ($url); # Pass the requested URL to the downloader. my $request = HTTP::Request->new(GET => $url); @@ -395,9 +388,6 @@ sub downloadruleset ($) { # Check if the server responds with 304 (Not Modified). } elsif ($response->code == 304) { - # Log to syslog. - &_log_to_syslog("Ruleset is up-to-date, no update required."); - # Return "not modified". return "not modified"; @@ -406,9 +396,6 @@ sub downloadruleset ($) { # Obtain error. my $error = $response->content; - # Log error message. - &_log_to_syslog("Unable to download the ruleset. \($error\)"); - # Return the error message from response.. return "$error"; } @@ -438,10 +425,6 @@ sub downloadruleset ($) { # Check if both file sizes match. if (($remote_filesize) && ($remote_filesize ne $local_filesize)) { - # Log error message. - &_log_to_syslog("Unable to completely download the ruleset. "); - &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. "); - # Delete temporary file. unlink("$tmpfile"); @@ -449,18 +432,6 @@ sub downloadruleset ($) { return "incomplete download"; } - # Check if a file name could be obtained. - unless ($dl_rulesfile) { - # Log error message. - &_log_to_syslog("Unable to store the downloaded rules file. "); - - # Delete downloaded temporary file. - unlink("$tmpfile"); - - # Return "1" - false. - return 1; - } - # Overwrite the may existing rulefile or tarball with the downloaded one. move("$tmpfile", "$dl_rulesfile"); From 65e3aef5835a5e681bdd2af292e4c547c0d196d0 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:17:06 +0100 Subject: [PATCH 11/77] ids-functionsn.pl: Remove logging calls when checking free diskspace. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index f4541ce77..b2a3a7390 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -255,11 +255,8 @@ sub checkdiskspace () { # Check if the available disk space is more than 300MB. if ($available < 300) { - # Log error to syslog. - &_log_to_syslog("Not enough free disk space on /var. Only $available MB from 300 MB available."); - - # Exit function and return "1" - False. - return 1; + # Exit function and return the available disk space. + return $available; } } } From d1f7542659cc7ecaaad551f813b0cb32a4734351 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:18:38 +0100 Subject: [PATCH 12/77] update-ids-ruleset: Add function to iherit with the syslog daemon. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index 8c5fd37a1..3c5cf583b 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -26,6 +26,9 @@ require '/var/ipfire/general-functions.pl'; require "${General::swroot}/ids-functions.pl"; require "${General::swroot}/lang.pl"; +# Load perl module to talk to the kernel syslog. +use Sys::Syslog qw(:DEFAULT setlogsock); + # Variable to store if the process has written a lockfile. my $locked; @@ -45,6 +48,9 @@ if ( $> == 0 ) { POSIX::setuid( $uid ); } +# Establish the connection to the syslog service. +openlog('oinkmaster', 'cons,pid', 'user'); + # Check if the IDS lock file exists. # In this case the WUI or another instance currently is altering the # ruleset. @@ -126,9 +132,24 @@ if(&IDS::ids_is_running()) { &IDS::call_suricatactrl("reload"); } -# Custom END declaration to release a IDS page lock -# when the script has created one. +# +# Tiny function to sent the error message to the syslog. +# +sub _log_to_syslog($) { + my ($message) = @_; + + # The syslog function works best with an array based input, + # so generate one before passing the message details to syslog. + my @syslog = ("ERR", "$message"); + + # Send the log message. + syslog(@syslog); +} + END { + # Close connection to syslog. + closelog(); + # Check if a lock has been requested. if ($locked) { # Unlock the IDS page. From c9c3eadbbffeee8a4f46365e917f619939dee9f1 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:22:08 +0100 Subject: [PATCH 13/77] update-ids-ruleset: Add logging for various events. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 42 ++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index 3c5cf583b..267ccdb25 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -56,7 +56,7 @@ openlog('oinkmaster', 'cons,pid', 'user'); # ruleset. if (-f "$IDS::ids_page_lock_file") { # Store notice to the syslog. - &IDS::_log_to_syslog("Another process currently is altering the IDS ruleset."); + &_log_to_syslog(" Autoupdate skipped - Another process was altering the IDS ruleset."); # Exit. exit 0; @@ -101,23 +101,37 @@ foreach my $id (keys %providers) { # Skip the provider if autoupdate is not enabled. next unless($autoupdate_status eq "enabled"); + # Log notice about update. + &_log_to_syslog(" Performing update for $provider."); + # Call the download function and gather the new ruleset for the current processed provider. - if(&IDS::downloadruleset($provider)) { - # Store error message for displaying in the WUI. - &IDS::_store_error_message("$provider: $Lang::tr{'could not download latest updates'}"); + my $return = &IDS::downloadruleset($provider); - # Unlock the IDS page. - &IDS::unlock_ids_page(); + # Check if we got a return code. + if ($return) { + # Handle different return codes. + if ($return eq "not_modified") { + # Log notice to syslog. + &_log_to_syslog(" Skipping $provider - The ruleset is up-to-date"); - # Exit. - exit 0; + } elsif ($return eq "no url") { + # Log error to the syslog. + &_log_to_syslog(" Could not determine a download URL for $provider."); + + } else { + # Log error to syslog. + &_log_to_syslog(" $return"); + } + } else { + # Log successfull update. + &_log_to_syslog(" Successfully updated ruleset for $provider."); + + # Get path and name of the stored rules file or archive. + my $stored_file = &IDS::_get_dl_rulesfile($provider); + + # Set correct ownership for the downloaded tarball. + &IDS::set_ownership("$stored_file"); } - - # Get path and name of the stored rules file or archive. - my $stored_file = &IDS::_get_dl_rulesfile($provider); - - # Set correct ownership for the downloaded tarball. - &IDS::set_ownership("$stored_file"); } # Call oinkmaster to alter the ruleset. From 9a3f9c2b234457e6cfda54f7ee3746781ba503b5 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:22:50 +0100 Subject: [PATCH 14/77] update-ids-ruleset: Log and abort if to less free disk space is available. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index 267ccdb25..9453f8f8e 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -65,19 +65,19 @@ if (-f "$IDS::ids_page_lock_file") { # Check if the red device is active. unless (-e "${General::swroot}/red/active") { # Store notice in the syslog. - &IDS::_log_to_syslog("The system is offline."); - - # Store error message for displaying in the WUI. - &IDS::_store_error_message("$Lang::tr{'could not download latest updates'} - $Lang::tr{'system is offline'}"); + &_log_to_syslog(" Could not update any ruleset - The system is offline."); # Exit. exit 0; } # Check if enought free disk space is availabe. -if(&IDS::checkdiskspace()) { - # Store the error message for displaying in the WUI. - &IDS::_store_error_message("$Lang::tr{'not enough disk space'}"); +my $return = &IDS::checkdiskspace(); + +# Handle error. +if ($return) { + # Store error in syslog. + &_log_to_syslog(" Not enough free disk space, only $return of 300MB are available."); # Exit. exit 0; From 74019d3044117bc84646fec22e6a88833a131790 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:23:44 +0100 Subject: [PATCH 15/77] update-ids-ruleset: Skip providers which are not enabled. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index 9453f8f8e..f87eb1926 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -96,8 +96,12 @@ $locked = "1"; foreach my $id (keys %providers) { # Assign some nice variabled. my $provider = $providers{$id}[0]; + my $enabled_status = $providers{$id}[2]; my $autoupdate_status = $providers{$id}[3]; + # Skip the provider if it is not enabled. + next unless($enabled_status eq "enabled"); + # Skip the provider if autoupdate is not enabled. next unless($autoupdate_status eq "enabled"); From ff780d8b3fa6b91fe9d8560684232381b81b5498 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 11:27:01 +0100 Subject: [PATCH 16/77] update-ids-ruleset: Fix typo in return code. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index f87eb1926..52df22a76 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -114,7 +114,7 @@ foreach my $id (keys %providers) { # Check if we got a return code. if ($return) { # Handle different return codes. - if ($return eq "not_modified") { + if ($return eq "not modified") { # Log notice to syslog. &_log_to_syslog(" Skipping $provider - The ruleset is up-to-date"); From e0eb5bc737fa807a574b4f5bf5c42977d55201fb Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 15:33:09 +0100 Subject: [PATCH 17/77] ids-functions.pl: Add get_providers_mode() function. This function is used to gather the modes of the configured providers and return them as hash. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 36 +++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index b2a3a7390..2aab57b41 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -234,6 +234,42 @@ sub get_enabled_providers () { return @enabled_providers; } +# +## Function to get a hash of provider handles and their configured modes (IDS/IPS). +# +sub get_providers_mode () { + my %used_providers = (); + + # Hash to store the providers and their configured modes. + my %providers_mode = (); + + # Read-in the providers config file. + &General::readhasharray("$providers_settings_file", \%used_providers); + + # Loop through the hash of used_providers. + foreach my $id (keys %used_providers) { + # Skip disabled providers. + next unless ($used_providers{$id}[3] eq "enabled"); + + # Grab the provider handle. + my $provider = "$used_providers{$id}[0]"; + + # Grab the provider mode. + my $mode = "$used_providers{$id}[4]"; + + # Fall back to IDS if no mode could be obtained. + unless($mode) { + $mode = "IDS"; + } + + # Add details to provider_modes hash. + $providers_mode{$provider} = $mode; + } + + # Return the hash. + return %providers_mode; +} + # ## Function for checking if at least 300MB of free disk space are available ## on the "/var" partition. From e246285af4c98647217fe96a48d794f959ebf3d8 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 15:34:10 +0100 Subject: [PATCH 18/77] ids-functions.pl: Add private function to obtain the sid and rev of a rule. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 2aab57b41..5e2345244 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -926,6 +926,27 @@ sub _get_dl_rulesfile($) { return $rulesfile; } +# +## Private function to obtain the sid and rev of a rule. +# +## Returns an array with the sid as first and the rev as second value. +# +sub _get_sid_and_rev ($) { + my ($line) = @_; + + my @ret; + + # Use regex to obtain the sid and rev. + if ($line =~ m/.*sid:\s*(.*?);.*rev:\s*(.*?);/) { + # Add the sid and rev to the array. + push(@ret, $1); + push(@ret, $2); + } + + # Return the array. + return @ret; +} + # ## Tiny function to delete the stored ruleset file or tarball for a given provider. # From 518cbdd38905ed7909f7dfe218957cbc828a004c Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 15:34:57 +0100 Subject: [PATCH 19/77] ids-functions.pl: Add get_provider_ruleset_modifications_file(). This function will obosolete the old oinkmaster modifications files. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 5e2345244..e86816019 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1550,6 +1550,18 @@ sub get_used_provider_rulesfile_file ($) { return $filename; } +# +## Tiny function to generate the full path and name for the file which stores the modifications of a ruleset. +# +sub get_provider_ruleset_modifications_file($) { + my ($provider) = @_; + + my $filename = "$settingsdir/$provider\-modifications"; + + # Return the filename. + return $filename; +} + # ## Function to generate and write the file for modify the ruleset. # From 5a6c7bbe85a24faddf3c5f495d28a6ae6004514f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 15:44:37 +0100 Subject: [PATCH 20/77] ids-functions.pl: Add process_ruleset() function. This function is going to replace the part which currently the oinkmaster.pl script does. It will read in the extracted ruleset, remove duplicates and alter the rules to alert or drop in case they match. Also rules will be enabled or disabled if the used requested this. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 118 ++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index e86816019..1e3cd4403 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -682,6 +682,124 @@ sub oinkmaster () { &cleanup_tmp_directory(); } +# +## Function to alter the ruleset. +# +sub process_ruleset(@) { + my (@providers) = @_; + + # Hash to store the configured provider modes. + my %providers_mode = &get_providers_mode(); + + # Array to store the extracted rulefile from the temporary rules directory. + my @extracted_rulefiles; + + # Get names of the extracted raw rulefiles. + opendir(DIR, $tmp_rules_directory) or die "Could not read from $tmp_rules_directory. $!\n"; + while (my $file = readdir(DIR)) { + # Ignore single and double dotted files. + next if $file =~ /^\.\.?$/; + + # Add file to the array of extracted files. + push(@extracted_rulefiles, $file); + } + + # Close directory handle. + closedir(DIR); + + # Loop through the array of providers. + foreach my $provider (@providers) { + # Hash to store the obtained SIDs and REV of each provider. + my %rules = (); + + # Hash which holds modifications to apply to the rules. + my %modifications = (); + + # Loop through the array of extraced rulefiles. + foreach my $file (@extracted_rulefiles) { + # Skip file if it does not belong to the current processed provider. + next unless ($file =~ m/^$provider/); + + # Open the rulefile. + open(FILE, "$tmp_rules_directory/$file") or die "Could not read $tmp_rules_directory/$file. $!\n"; + + # Loop through the file content. + while (my $line = ) { + # Skip blank lines. + next if ($line =~ /^\s*$/); + + # Call function to get the sid and rev of the rule. + my ($sid, $rev) = &_get_sid_and_rev($line); + + # Skip rule if a sid with a higher rev already has added to the rules hash. + next if ($rev le $rules{$sid}); + + # Add the new or rule with higher rev to the hash of rules. + $rules{$sid} = $rev; + } + + # Close file handle. + close(FILE); + } + + # Get filename which contains the ruleset modifications for this provider. + my $modification_file = &get_provider_ruleset_modifications_file($provider); + + # Read file which holds the modifications of the ruleset for the current provider. + &General::readhash($modification_file, \%modifications) if (-f $modification_file); + + # Loop again through the array of extracted rulesfiles. + foreach my $file (@extracted_rulefiles) { + # Skip the file if it does not belong to the current provider. + next unless ($file =~ m/^$provider/); + + # Open the rulefile for writing. + open(RULEFILE, ">", "$rulespath/$file") or die "Could not write to file $rulespath/$file. $!\n"; + + # Open the rulefile for reading. + open(TMP_RULEFILE, "$tmp_rules_directory/$file") or die "Could not read $tmp_rules_directory/$file. $!\n"; + + # Loop through the raw temporary rulefile. + while (my $line = ) { + # Get the sid and rev of the rule. + my ($sid, $rev) = &_get_sid_and_rev($line); + + # Check if the current rule is obsoleted by a newer one. + # + # In this case the rev number in the rules hash is higher than the current one. + next if ($rev lt $rules{$sid}); + + # Check if the rule should be enabled or disabled. + if ($modifications{$sid} eq "on") { + # Drop the # at the start of the line. + $line =~ s/^\#//; + } elsif ($modifications{$sid} eq "off") { + # Add a # at the start of the line to disable the rule. + $line = "#$line" unless ($line =~ /^#/); + } + + # Check if the Provider is set so IPS mode. + if ($providers_mode{$provider} eq "IPS") { + # Replacements for sourcefire rules. + $line =~ s/^#\s*(?:alert|drop)(.+policy balanced-ips alert)/alert${1}/; + $line =~ s/^#\s*(?:alert|drop)(.+policy balanced-ips drop)/drop${1}/; + + # Replacements for generic rules. + $line =~ s/^(#?)\s*(?:alert|drop)/${1}drop/; + $line =~ s/^(#?)\s*drop(.+flowbits:noalert;)/${1}alert${2}/; + } + + # Write line / rule to the target rule file. + print RULEFILE "$line"; + } + + # Close filehandles. + close(RULEFILE); + close(TMP_RULEFILE); + } + } +} + # ## Function to merge the classifications for a given amount of providers and write them ## to the classifications file. From 794469483f26e514e13648a07483d19e2372ecb7 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 15:47:52 +0100 Subject: [PATCH 21/77] ids-functions.pl: Replace call of external oinkmaster.pl to newly introduced process_ruleset function. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 1e3cd4403..d318f464a 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -650,27 +650,8 @@ sub oinkmaster () { &extractruleset($provider); } - # Establish the connection to the syslog service. - openlog('oinkmaster', 'cons,pid', 'user'); - - # Call oinkmaster to generate ruleset. - open(OINKMASTER, "/usr/local/bin/oinkmaster.pl -s -u dir://$tmp_rules_directory -C $settingsdir/oinkmaster.conf -o $rulespath 2>&1 |") or die "Could not execute oinkmaster $!\n"; - - # Log output of oinkmaster to syslog. - while() { - # The syslog function works best with an array based input, - # so generate one before passing the message details to syslog. - my @syslog = ("INFO", "$_"); - - # Send the log message. - syslog(@syslog); - } - - # Close the pipe to oinkmaster process. - close(OINKMASTER); - - # Close the log handle. - closelog(); + # Call function to process the ruleset and do all modifications. + &process_ruleset(@enabled_providers); # Call function to merge the classification files. &merge_classifications(@enabled_providers); From 2deba6bf4a6b866713ee000a91457802101fa893 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 16:10:01 +0100 Subject: [PATCH 22/77] ids-functions.pl: Use "enabled/disabled" to mark if a rule should be altered. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index d318f464a..e09fe0485 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -751,10 +751,10 @@ sub process_ruleset(@) { next if ($rev lt $rules{$sid}); # Check if the rule should be enabled or disabled. - if ($modifications{$sid} eq "on") { + if ($modifications{$sid} eq "enabled") { # Drop the # at the start of the line. $line =~ s/^\#//; - } elsif ($modifications{$sid} eq "off") { + } elsif ($modifications{$sid} eq "disabled") { # Add a # at the start of the line to disable the rule. $line = "#$line" unless ($line =~ /^#/); } From 9f353f8518b93fb1b4f76663088d36d321e8e3f2 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 16:11:12 +0100 Subject: [PATCH 23/77] ids.cgi: Use new backend to store the ruleset modifications of a provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 49 ++++++-------------------------------------- 1 file changed, 6 insertions(+), 43 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 76c2b99a9..8a52876de 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -339,12 +339,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { } } - # Open oinkmaster main include file for provider modifications. - open(OINKM_INCL_FILE, ">", "$IDS::oinkmaster_provider_includes_file") or die "Could not open $IDS::oinkmaster_provider_includes_file. $!\n"; - - # Print file header and notice about autogenerated file. - print OINKM_INCL_FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - # Get enabled providers. my @enabled_providers = &IDS::get_enabled_providers(); @@ -353,14 +347,11 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Hash to store the used-enabled and disabled sids. my %enabled_disabled_sids; - # Generate modified sids file name for the current processed provider. - my $providers_modified_sids_file = &IDS::get_oinkmaster_provider_modified_sids_file($provider); + # Get name of the file which holds the ruleset modification of the provider. + my $modifications_file = &IDS::get_provider_ruleset_modifications_file($provider); - # Check if a modified sids file for this provider exists. - if (-f $providers_modified_sids_file) { - # Read-in the file for enabled/disabled sids. - %enabled_disabled_sids = &IDS::read_enabled_disabled_sids_file($providers_modified_sids_file); - } + # Read-in modifications file, if exists. + &General::readhash("$modifications_file", \%enabled_disabled_sids) if (-f "$modifications_file"); # Loop through the hash of idsrules. foreach my $rulefile (keys %idsrules) { @@ -409,39 +400,11 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Check if the hash for enabled/disabled sids contains any entries. if (%enabled_disabled_sids) { - # Open providers modified sids file for writing. - open(PROVIDER_MOD_FILE, ">$providers_modified_sids_file") or die "Could not write to $providers_modified_sids_file. $!\n"; - - # Write header to the files. - print PROVIDER_MOD_FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - - # Loop through the hash. - foreach my $sid (keys %enabled_disabled_sids) { - # Check if the sid is enabled. - if ($enabled_disabled_sids{$sid} eq "enabled") { - # Print the sid to the enabled_sids file. - print PROVIDER_MOD_FILE "enablesid $sid\n"; - # Check if the sid is disabled. - } elsif ($enabled_disabled_sids{$sid} eq "disabled") { - # Print the sid to the disabled_sids file. - print PROVIDER_MOD_FILE "disablesid $sid\n"; - # Something strange happende - skip the current sid. - } else { - next; - } - } - - # Close file handle for the providers modified sids file. - close(PROVIDER_MOD_FILE); - - # Add the file to the oinkmasters include file. - print OINKM_INCL_FILE "include $providers_modified_sids_file\n"; + # Write the modifications file. + &General::writehash("$modifications_file", \%enabled_disabled_sids); } } - # Close the file handle after writing. - close(OINKM_INCL_FILE); - # Handle enabled / disabled rulefiles. # From 4c98be8bd21c95b9bb576e211e633bf507388234 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 16:33:20 +0100 Subject: [PATCH 24/77] ids.cgi: Use new provider modifications backend. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 8a52876de..946df465f 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -513,19 +513,16 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # for this provider. &IDS::write_used_provider_rulefiles_file($provider); - # Call function to get the path and name for the given providers - # oinkmaster modified sids file. - my $provider_modified_sids_file = &IDS::get_oinkmaster_provider_modified_sids_file($provider); + # Call function to get the path and name for file which holds the ruleset modifications + # for the given provider. + my $modifications_file = &IDS::get_provider_ruleset_modifications_file($provider); # Check if the file exists. - if (-f $provider_modified_sids_file) { + if (-f $modifications_file) { # Remove the file, as requested. - unlink("$provider_modified_sids_file"); + unlink("$modifications_file"); } - # Alter the oinkmaster provider includes file and remove the provider. - &IDS::alter_oinkmaster_provider_includes_file("remove", $provider); - # Regenerate ruleset. &IDS::oinkmaster(); @@ -880,10 +877,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Write the main providers include file. &IDS::write_main_used_rulefiles_file(@enabled_providers); - # Call function to alter the oinkmasters provider includes file and - # add or remove the provider. - &IDS::alter_oinkmaster_provider_includes_file($provider_includes_action, $provider_handle); - # Check if oinkmaster has to be executed. if ($oinkmaster eq "True") { # Lock the webpage and print message. @@ -945,18 +938,15 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { unlink("$provider_used_rulefile"); # Call function to get the path and name for the given providers - # oinkmaster modified sids file. - my $provider_modified_sids_file = &IDS::get_oinkmaster_provider_modified_sids_file($provider); + # ruleset modifications file.. + my $modifications_file = &IDS::get_provider_ruleset_modifications_file($provider); # Check if the file exists. - if (-f $provider_modified_sids_file) { + if (-f $modifications_file) { # Remove the file, which is not longer needed. - unlink("$provider_modified_sids_file"); + unlink("$modifications_file"); } - # Alter the oinkmaster provider includes file and remove the provider. - &IDS::alter_oinkmaster_provider_includes_file("remove", $provider); - # Regenerate ruleset. &IDS::oinkmaster(); @@ -1846,12 +1836,12 @@ sub show_additional_provider_actions() { # Assign variable for provider handle. my $provider = "$used_providers{$cgiparams{'ID'}}[0]"; - # Call function to get the path and name for the given providers - # oinkmaster modified sids file. - my $provider_modified_sids_file = &IDS::get_oinkmaster_provider_modified_sids_file($provider); + # Call function to get the path and name for the given provider + # ruleset modifications file. + my $modifications_file = &IDS::get_provider_ruleset_modifications_file($provider); # Disable the reset provider button if no provider modified sids file exists. - unless (-f $provider_modified_sids_file) { + unless (-f $modifications_file) { $disabled = "disabled"; } From 443ad51d1c33550eafc62320865046510b7be8fc Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 16:52:19 +0100 Subject: [PATCH 25/77] ids.cgi: Allow to configure IDS/IPS mode individually for each provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 946df465f..be16a4eb0 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -696,6 +696,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { my $provider = $cgiparams{'PROVIDER'}; my $subscription_code = $cgiparams{'SUBSCRIPTION_CODE'}; my $status_autoupdate; + my $mode; # Handle autoupdate checkbox. if ($cgiparams{'ENABLE_AUTOUPDATE'} eq "on") { @@ -704,6 +705,13 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { $status_autoupdate = "disabled"; } + # Handle monitor traffic only checkbox. + if ($cgiparams{'MONITOR_TRAFFIC_ONLY'} eq "on") { + $mode = "IDS"; + } else { + $mode = "IPS"; + } + # Check if we are going to add a new provider. if ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'add'}") { # Loop through the hash of used providers. @@ -766,7 +774,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { } # Add/Modify the entry to/in the used providers hash.. - $used_providers{$id} = ["$provider", "$subscription_code", "$status_autoupdate", "$status"]; + $used_providers{$id} = ["$provider", "$subscription_code", "$status_autoupdate", "$status", "$mode"]; # Write the changed hash to the providers settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); @@ -1029,9 +1037,6 @@ sub show_mainpage() { $checked{'ENABLE_IDS'}{'off'} = ''; $checked{'ENABLE_IDS'}{'on'} = ''; $checked{'ENABLE_IDS'}{$idssettings{'ENABLE_IDS'}} = "checked='checked'"; - $checked{'MONITOR_TRAFFIC_ONLY'}{'off'} = ''; - $checked{'MONITOR_TRAFFIC_ONLY'}{'on'} = ''; - $checked{'MONITOR_TRAFFIC_ONLY'}{$idssettings{'MONITOR_TRAFFIC_ONLY'}} = "checked='checked'"; $selected{'AUTOUPDATE_INTERVAL'}{'off'} = ''; $selected{'AUTOUPDATE_INTERVAL'}{'daily'} = ''; $selected{'AUTOUPDATE_INTERVAL'}{'weekly'} = ''; @@ -1102,8 +1107,6 @@ print < $Lang::tr{'ids enable'} - -  $Lang::tr{'ids monitor traffic only'} @@ -1698,6 +1701,12 @@ END $checked{'ENABLE_AUTOUPDATE'} = "checked='checked'"; } + # Check if the monitor traffic only mode is set for this provider. + if ($used_providers{$cgiparams{'ID'}}[4] eq "IDS") { + # Set the checkbox to be checked. + $checked{'MONITOR_TRAFFIC_ONLY'} = "checked='checked'"; + } + # Display section to force an rules update and to reset the provider. &show_additional_provider_actions(); @@ -1795,9 +1804,13 @@ print < - +  $Lang::tr{'ids enable automatic updates'} + + +  $Lang::tr{'ids monitor traffic only'} + From 849fc8ea15a861a97f2e4d9c74804115fd15ecf5 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 18:08:49 +0100 Subject: [PATCH 26/77] ids-functions.pl: Drop oinkmaster related functions and declarations. They are not longer needed and safely can be dropped. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 113 -------------------------------- 1 file changed, 113 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index e09fe0485..ff84e1ad3 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -77,12 +77,6 @@ our $dns_servers_file = "$settingsdir/suricata-dns-servers.yaml"; # File where the HTTP ports definition is stored. our $http_ports_file = "$settingsdir/suricata-http-ports.yaml"; -# File which contains includes for provider specific rule modifications. -our $oinkmaster_provider_includes_file = "$settingsdir/oinkmaster-provider-includes.conf"; - -# File which contains wheater the rules should be changed. -our $modify_sids_file = "$settingsdir/oinkmaster-modify-sids.conf"; - # File which stores the configured IPS settings. our $ids_settings_file = "$settingsdir/settings"; @@ -177,8 +171,6 @@ my %tr_app_layer_proto = ( # sub check_and_create_filelayout() { # Check if the files exist and if not, create them. - unless (-f "$oinkmaster_provider_includes_file") { &create_empty_file($oinkmaster_provider_includes_file); } - unless (-f "$modify_sids_file") { &create_empty_file($modify_sids_file); } unless (-f "$suricata_used_providers_file") { &create_empty_file($suricata_used_providers_file); } unless (-f "$suricata_default_rulefiles_file") { &create_empty_file($suricata_default_rulefiles_file); } unless (-f "$ids_settings_file") { &create_empty_file($ids_settings_file); } @@ -1062,65 +1054,6 @@ sub drop_dl_rulesfile ($) { } } -# -## Tiny function to get/generate the full path and filename for the providers oinkmaster -## modified sids file. -# -sub get_oinkmaster_provider_modified_sids_file ($) { - my ($provider) = @_; - - # Generate the filename. - my $filename = "$settingsdir/oinkmaster-$provider-modified-sids.conf"; - - # Return the filename. - return $filename; -} - -# -## Function to directly altering the oinkmaster provider includes file. -## -## Requires tha acition "remove" or "add" and a provider handle. -# -sub alter_oinkmaster_provider_includes_file ($$) { - my ($action, $provider) = @_; - - # Call function to get the path and name for the given providers - # oinkmaster modified sids file. - my $provider_modified_sids_file = &get_oinkmaster_provider_modified_sids_file($provider); - - # Open the file for reading.. - open (FILE, $oinkmaster_provider_includes_file) or die "Could not read $oinkmaster_provider_includes_file. $!\n"; - - # Read-in file content. - my @lines = ; - - # Close file after reading. - close(FILE); - - # Re-open the file for writing. - open(FILE, ">", $oinkmaster_provider_includes_file) or die "Could not write to $oinkmaster_provider_includes_file. $!\n"; - - # Loop through the file content. - foreach my $line (@lines) { - # Remove newlines. - chomp($line); - - # Skip line if we found our given provider and the action should be remove. - next if (($line =~ /$provider/) && ($action eq "remove")); - - # Write the read-in line back to the file. - print FILE "$line\n"; - } - - # Check if the file exists and add the provider if requested. - if ((-f $provider_modified_sids_file) && ($action eq "add")) { - print FILE "include $provider_modified_sids_file\n"; - } - - # Close file handle. - close(FILE); -} - # ## Function to read-in the given enabled or disables sids file. # @@ -1661,52 +1594,6 @@ sub get_provider_ruleset_modifications_file($) { return $filename; } -# -## Function to generate and write the file for modify the ruleset. -# -sub write_modify_sids_file() { - # Get configured settings. - my %idssettings=(); - &General::readhash("$ids_settings_file", \%idssettings); - - # Open modify sid's file for writing. - open(FILE, ">$modify_sids_file") or die "Could not write to $modify_sids_file. $!\n"; - - # Write file header. - print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - - # Check if the traffic only should be monitored. - unless($idssettings{'MONITOR_TRAFFIC_ONLY'} eq 'on') { - # Suricata is in IPS mode, which means that the rule actions have to be changed - # from 'alert' to 'drop', however not all rules should be changed. Some rules - # exist purely to set a flowbit which is used to convey other information, such - # as a specific type of file being downloaded, to other rulewhich then check for - # malware in that file. Rules which fall into the first category should stay as - # alert since not all flows of that type contain malware. - - # These types of rulesfiles contain meta-data which gives the action that should - # be used when in IPS mode. Do the following: - # - # 1. Disable all rules and set the action to 'drop' - # 2. Set the action back to 'alert' if the rule contains 'flowbits:noalert;' - # This should give rules not in the policy a reasonable default if the user - # manually enables them. - # 3. Enable rules and set actions according to the meta-data strings. - - my $policy = 'balanced'; # Placeholder to allow policy to be changed. - - print FILE < Date: Sat, 26 Mar 2022 11:54:19 +0100 Subject: [PATCH 27/77] ids.cgi: Drop last fragments from old modify sids backend. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index be16a4eb0..0907ba15a 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -613,24 +613,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Generate file to store the HTTP ports. &IDS::generate_http_ports_file(); - # Write the modify sid's file and pass the taken ruleaction. - &IDS::write_modify_sids_file(); - - # Check if "MONITOR_TRAFFIC_ONLY" has been changed. - if($cgiparams{'MONITOR_TRAFFIC_ONLY'} ne $oldidssettings{'MONITOR_TRAFFIC_ONLY'}) { - # Check if at least one provider is enabled. - if (@enabled_providers) { - # Lock the webpage and print message. - &working_notice("$Lang::tr{'ids working'}"); - - # Call oinkmaster to alter the ruleset. - &IDS::oinkmaster(); - - # Set reload_page to "True". - $reload_page="True"; - } - } - # Check if the IDS currently is running. if(&IDS::ids_is_running()) { # Check if ENABLE_IDS is set to on. From 81144407528319a53fd0e8ea6852158c56ab7612 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 20 Mar 2022 18:59:42 +0100 Subject: [PATCH 28/77] convert-ids-modification-files: New converter. This converter is responsible to convert the old oinkmaster modification files into the new files and format. Signed-off-by: Stefan Schantl --- .../suricata/convert-ids-modification-files | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 config/suricata/convert-ids-modification-files diff --git a/config/suricata/convert-ids-modification-files b/config/suricata/convert-ids-modification-files new file mode 100644 index 000000000..555deaf18 --- /dev/null +++ b/config/suricata/convert-ids-modification-files @@ -0,0 +1,80 @@ +#!/usr/bin/perl +############################################################################### +# # +# IPFire.org - A linux based firewall # +# Copyright (C) 2021 IPFire Development Team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +use strict; + +require '/var/ipfire/general-functions.pl'; +require '/var/ipfire/ids-functions.pl'; + +# Exit if there is no main oinkmaster config file anymore. +exit 0 unless (-f "$IDS::settingsdir/oinkmaster.conf"); + +# Get all supported providers. +my @providers = &IDS::get_ruleset_providers(); + +# Loop through the array of providers. +foreach my $provider (@providers) { + my %modifications = (); + + # Generate old filename which hold the ruleset modifications. + my $old_modifications_file = "$IDS::settingsdir/oinkmaster\-$provider\-modified-sids.conf"; + + # Skip provider if there is no modifications file. + next unless (-f $old_modifications_file); + + # Open modifications file. + open(FILE, "$old_modifications_file"); + + # Read-in file content. + my @file = ; + + # Close file handle. + close(FILE); + + # Loop through the file content. + foreach my $line (@file) { + chomp($line); + + # Split line and assign to an temporary array. + my @tmp = split(/ /, $line); + + # Assign nice human-readable variables. + my $action = $tmp[0]; + my $sid = $tmp[1]; + + # Process stored rule action and assign to the modifications hash. + if ($action eq "enablesid") { + $modifications{$sid} = "enabled"; + + } elsif ($action eq "disablesid") { + $modifications{$sid} = "disabled"; + } + } + + # Get new filename which will hold the ruleset modifications for this provider. + my $new_modifications_file = &IDS::get_provider_ruleset_modifications_file($provider); + + # Write new modifications file. + &General::writehash("$new_modifications_file", \%modifications); + + # Set correct ownership for the new modifications file. + &IDS::set_ownership("$new_modifications_file"); +} From fa7663a1b594dcfd4bf542eb34a0869d5280e38f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 26 Mar 2022 12:26:35 +0100 Subject: [PATCH 29/77] ids.cgi: Remove newly added provider if the rules could not be downloaded. When adding a new provider and in case the rules file or tarball can not be downloaded, the provider remains as configured. To avoid that, the provider needs to be removed again. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 0907ba15a..07fcae9e3 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -805,6 +805,9 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Perform a reload of the page. &reload(); + } else { + # Remove the configured provider again. + &remove_provider($id); } } From 8d6714edc8c957214506bc483bc51edc06c94554 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 9 Apr 2022 13:11:18 +0200 Subject: [PATCH 30/77] ids-functions.pl: Change backend to use one file to load the used rulefiles. Suricata seems to struggle when using multiple and/or nested includes in the same config section. This results in a only partially loaded confguration where not all rulefiles are loaded and used. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 160 +++++++++++--------------------- config/suricata/suricata.yaml | 5 +- 2 files changed, 53 insertions(+), 112 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index ff84e1ad3..cf896f340 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -63,10 +63,7 @@ use Sys::Syslog qw(:DEFAULT setlogsock); our $settingsdir = "${General::swroot}/suricata"; # File where the main file for providers ruleset inclusion exists. -our $suricata_used_providers_file = "$settingsdir/suricata-used-providers.yaml"; - -# File for static ruleset inclusions. -our $suricata_default_rulefiles_file = "$settingsdir/suricata-default-rules.yaml"; +our $suricata_used_rulesfiles_file = "$settingsdir/suricata-used-rulesfiles.yaml"; # File where the addresses of the homenet are stored. our $homenet_file = "$settingsdir/suricata-homenet.yaml"; @@ -171,8 +168,7 @@ my %tr_app_layer_proto = ( # sub check_and_create_filelayout() { # Check if the files exist and if not, create them. - unless (-f "$suricata_used_providers_file") { &create_empty_file($suricata_used_providers_file); } - unless (-f "$suricata_default_rulefiles_file") { &create_empty_file($suricata_default_rulefiles_file); } + unless (-f "$suricata_used_rulesfiles_file") { &create_empty_file($suricata_used_rulesfiles_file); } unless (-f "$ids_settings_file") { &create_empty_file($ids_settings_file); } unless (-f "$providers_settings_file") { &create_empty_file($providers_settings_file); } unless (-f "$whitelist_file" ) { &create_empty_file($whitelist_file); } @@ -1446,83 +1442,19 @@ sub generate_http_ports_file() { } # -## Function to generate and write the file for used rulefiles file for a given provider. +## Function to write the file that contains the rulefiles which are loaded by suricaa. ## -## The function requires as first argument a provider handle, and as second an array with files. +## This function requires an array of used provider handles. # -sub write_used_provider_rulefiles_file($@) { - my ($provider, @files) = @_; - - # Get the path and file for the provider specific used rulefiles file. - my $used_provider_rulesfile_file = &get_used_provider_rulesfile_file($provider); - - # Open file for used rulefiles. - open (FILE, ">", "$used_provider_rulesfile_file") or die "Could not write to $used_provider_rulesfile_file. $!\n"; - - # Write yaml header to the file. - print FILE "%YAML 1.1\n"; - print FILE "---\n\n"; - - # Write header to file. - print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - - # Loop through the array of given files. - foreach my $file (@files) { - # Check if the given filename exists and write it to the file of used rulefiles. - if(-f "$rulespath/$file") { - print FILE " - $file\n"; - } - } - - # Close file after writing. - close(FILE); -} - -# -## Function to write the main file for provider rulesfiles inclusions. -## -## This function requires an array of provider handles. -# -sub write_main_used_rulefiles_file (@) { +sub write_used_rulefiles_file (@) { my (@providers) = @_; - # Call function to write the static rulefiles file. - &_write_default_rulefiles_file(); - - # Open file for used rulefils inclusion. - open (FILE, ">", "$suricata_used_providers_file") or die "Could not write to $suricata_used_providers_file. $!\n"; - - # Write yaml header to the file. - print FILE "%YAML 1.1\n"; - print FILE "---\n\n"; - - # Write header to file. - print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - - # Loop through the list of given providers. - foreach my $provider (@providers) { - # Call function to get the providers used rulefiles file. - my $filename = &get_used_provider_rulesfile_file($provider); - - # Check if the file exists and write it into the used rulefiles file. - if (-f $filename) { - # Print the provider to the file. - print FILE "include\: $filename\n"; - } - } - - # Close the filehandle after writing. - close(FILE); -} - -sub _write_default_rulefiles_file () { - # Get enabled application layer protocols. + # Get the enabled application layer protocols. my @enabled_app_layer_protos = &get_suricata_enabled_app_layer_protos(); - # Open file. - open (FILE, ">", $suricata_default_rulefiles_file) or die "Could not write to $suricata_default_rulefiles_file. $!\n"; + # Open the file. + open (FILE, ">", $suricata_used_rulesfiles_file) or die "Could not write to $suricata_used_rulesfiles_file. $!\n"; - # Write yaml header to the file. print FILE "%YAML 1.1\n"; print FILE "---\n\n"; @@ -1566,17 +1498,38 @@ sub _write_default_rulefiles_file () { } } + # Loop through the array of enabled providers. + foreach my $provider (@providers) { + # Get the used rulefile for this provider. + my @used_rulesfiles = &get_provider_used_rulesfiles($provider); + + # Check if there are + if(@used_rulesfiles) { + # Add notice to the file. + print FILE "\n#Used Rulesfiles for provider $provider.\n"; + + # Loop through the array of used rulefiles. + foreach my $enabled_rulesfile (@used_rulesfiles) { + # Generate name and full path to the rulesfile. + my $rulesfile = "$rulespath/$enabled_rulesfile"; + + # Write the ruelsfile name to the file. + print FILE " - $rulesfile\n"; + } + } + } + # Close the file handle close(FILE); } # -## Tiny function to generate the full path and name for the used_provider_rulesfile file of a given provider. +## Tiny function to generate the full path and name for the file which stores the used rulefiles of a given provider. # -sub get_used_provider_rulesfile_file ($) { +sub get_provider_used_rulesfiles_file ($) { my ($provider) = @_; - my $filename = "$settingsdir/suricata\-$provider\-used\-rulefiles.yaml"; + my $filename = "$settingsdir/$provider\-used\-rulesfiles"; # Return the gernerated file. return $filename; @@ -1909,44 +1862,35 @@ sub get_red_address() { # ## Function to get the used rules files of a given provider. # -sub read_used_provider_rulesfiles($) { +sub get_provider_used_rulesfiles($) { my ($provider) = @_; + # Hash to store the used rulefiles of the provider. + my %provider_rulefiles = (); + # Array to store the used rulefiles. my @used_rulesfiles = (); - # Get the used rulesefile file for the provider. - my $rulesfile_file = &get_used_provider_rulesfile_file($provider); + # Get the filename which contains the used rulefiles for this provider. + my $used_rulesfiles_file = &get_provider_used_rulesfiles_file($provider); - # Check if the a used rulesfile exists for this provider. - if (-f $rulesfile_file) { - # Open the file or used rulefiles and read-in content. - open(FILE, $rulesfile_file) or die "Could not open $rulesfile_file. $!\n"; + # Read-in file, if it exists. + &General::readhash("$used_rulesfiles_file", \%provider_rulefiles) if (-f $used_rulesfiles_file); - while () { - # Assign the current line to a nice variable. - my $line = $_; + # Loop through the hash of rulefiles which does the provider offer. + foreach my $rulefile (keys %provider_rulefiles) { + # Skip disabled rulefiles. + next unless($provider_rulefiles{$rulefile} eq "enabled"); - # Remove newlines. - chomp($line); + # The General::readhash function does not allow dots as + # key value and limits the key "string" to the part before + # the dot, in case it contains one. + # + # So add the file extension for the rules file manually again. + $rulefile = "$rulefile.rules"; - # Skip comments. - next if ($line =~ /\#/); - - # Skip blank lines. - next if ($line =~ /^\s*$/); - - # Gather the rulefile. - if ($line =~ /.*- (.*)/) { - my $rulefile = $1; - - # Add the rulefile to the array of used rulesfiles. - push(@used_rulesfiles, $rulefile); - } - } - - # Close the file. - close(FILE); + # Add the enabled rulefile to the array of enabled rulefiles. + push(@used_rulesfiles, $rulefile); } # Return the array of used rulesfiles. diff --git a/config/suricata/suricata.yaml b/config/suricata/suricata.yaml index 6fbc7b3ee..03a7a83af 100644 --- a/config/suricata/suricata.yaml +++ b/config/suricata/suricata.yaml @@ -47,10 +47,7 @@ vars: default-rule-path: /var/lib/suricata rule-files: # Include enabled ruleset files from external file. - include: /var/ipfire/suricata/suricata-used-providers.yaml - - # Include default rules. - include: /var/ipfire/suricata/suricata-default-rules.yaml + include: /var/ipfire/suricata/suricata-used-rulesfiles.yaml classification-file: /usr/share/suricata/classification.config reference-config-file: /usr/share/suricata/reference.config From 30c4a9ff35117388ce3061ad44280967e1f4cf86 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 9 Apr 2022 14:46:39 +0200 Subject: [PATCH 31/77] ids.cgi: Adjust code to use new used-rulesfiles backend. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 110 ++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 74 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 07fcae9e3..1efead803 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -298,7 +298,7 @@ if ($cgiparams{'RULESET'}) { # Loop through the array of used providers. foreach my $provider (@enabled_providers) { # Gather used rulefiles. - my @used_rulesfiles = &IDS::read_used_provider_rulesfiles($provider); + my @used_rulesfiles = &IDS::get_provider_used_rulesfiles($provider); # Loop through the array of used rulesfiles. foreach my $rulefile (@used_rulesfiles) { @@ -315,30 +315,6 @@ if ($cgiparams{'RULESET'}) { # Save ruleset. if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { - # Arrays to store which rulefiles have been enabled and will be used. - my @enabled_rulefiles; - - # Store if a restart of suricata is required. - my $suricata_restart_required; - - # Loop through the hash of idsrules. - foreach my $rulefile(keys %idsrules) { - # Check if the state of the rulefile has been changed. - unless ($cgiparams{$rulefile} eq $idsrules{$rulefile}{'Rulefile'}{'State'}) { - # A restart of suricata is required to apply the changes of the used rulefiles. - $suricata_restart_required = 1; - } - - # Check if the rulefile is enabled. - if ($cgiparams{$rulefile} eq "on") { - # Add rulefile to the array of enabled rulefiles. - push(@enabled_rulefiles, $rulefile); - - # Drop item from cgiparams hash. - delete $cgiparams{$rulefile}; - } - } - # Get enabled providers. my @enabled_providers = &IDS::get_enabled_providers(); @@ -347,9 +323,15 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Hash to store the used-enabled and disabled sids. my %enabled_disabled_sids; + # Hash to store the enabled rulefiles for the current processed provider. + my %used_rulefiles; + # Get name of the file which holds the ruleset modification of the provider. my $modifications_file = &IDS::get_provider_ruleset_modifications_file($provider); + # Get the name of the file which contains the used rulefiles for this provider. + my $used_rulefiles_file = &IDS::get_provider_used_rulesfiles_file($provider); + # Read-in modifications file, if exists. &General::readhash("$modifications_file", \%enabled_disabled_sids) if (-f "$modifications_file"); @@ -364,6 +346,15 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Skip the rulefile if the vendor is not our current processed provider. next unless ($rulefile_vendor eq $provider); + # Check if the rulefile is enabled. + if ($cgiparams{$rulefile} eq "on") { + # Add the rulefile to the hash of enabled rulefiles of this provider. + $used_rulefiles{$rulefile} = "enabled"; + + # Drop item from cgiparams hash. + delete $cgiparams{$rulefile}; + } + # Loop through the single rules of the rulefile. foreach my $sid (keys %{$idsrules{$rulefile}}) { # Skip the current sid if it is not numeric. @@ -403,37 +394,13 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Write the modifications file. &General::writehash("$modifications_file", \%enabled_disabled_sids); } - } - # Handle enabled / disabled rulefiles. - # - - # Loop through the array of enabled providers. - foreach my $provider(@enabled_providers) { - # Array to store the rulefiles which belong to the current processed provider. - my @provider_rulefiles = (); - - # Loop through the array of enabled rulefiles. - foreach my $rulesfile (@enabled_rulefiles) { - # Split the rulefile name. - my @filename_parts = split(/-/, "$rulesfile"); - - # Assign vendor name for easy processings. - my $vendor = @filename_parts[0]; - - # Check if the rulesvendor is our current processed enabled provider. - if ("$vendor" eq "$provider") { - # Add the rulesfile to the array of provider rulesfiles. - push(@provider_rulefiles, $rulesfile); - } - - # Call function and write the providers used rulesfile file. - &IDS::write_used_provider_rulefiles_file($provider, @provider_rulefiles); - } + # Write the used rulefiles file. + &General::writehash("$used_rulefiles_file", \%used_rulefiles); } # Call function to generate and write the used rulefiles file. - &IDS::write_main_used_rulefiles_file(@enabled_providers); + &IDS::write_used_rulefiles_file(@enabled_providers); # Lock the webpage and print message. &working_notice("$Lang::tr{'ids apply ruleset changes'}"); @@ -443,14 +410,8 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Check if the IDS is running. if(&IDS::ids_is_running()) { - # Check if a restart of suricata is required. - if ($suricata_restart_required) { - # Call suricatactrl to perform the restart. - &IDS::call_suricatactrl("restart"); - } else { - # Call suricatactrl to perform a reload. - &IDS::call_suricatactrl("reload"); - } + # Call suricatactrl to perform a reload. + &IDS::call_suricatactrl("reload"); } # Reload page. @@ -503,15 +464,20 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Reset a provider to it's defaults. } elsif ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'ids reset provider'}") { + # Get enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + # Grab provider handle from cgihash. my $provider = $cgiparams{'PROVIDER'}; # Lock the webpage and print message. &working_notice("$Lang::tr{'ids apply ruleset changes'}"); - # Create new empty file for used rulefiles - # for this provider. - &IDS::write_used_provider_rulefiles_file($provider); + # Get the name of the file which contains the used rulefiles for this provider. + my $used_rulefiles_file = &IDS::get_provider_used_rulesfiles_file($provider); + + # Remove the file if it exists. + unlink("$used_rulefiles_file") if (-f "$used_rulefiles_file"); # Call function to get the path and name for file which holds the ruleset modifications # for the given provider. @@ -523,21 +489,21 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { unlink("$modifications_file"); } + # Write used rulesfiles file. + &IDS::write_used_rulefiles_file(@enabled_providers); + # Regenerate ruleset. &IDS::oinkmaster(); # Check if the IDS is running. if(&IDS::ids_is_running()) { - # Get enabled providers. - my @enabled_providers = &IDS::get_enabled_providers(); - # Get amount of enabled providers. my $amount = @enabled_providers; # Check if at least one enabled provider remains. if ($amount >= 1) { # Call suricatactrl to perform a reload. - &IDS::call_suricatactrl("restart"); + &IDS::call_suricatactrl("reload"); # Stop suricata if no enabled provider remains. } else { @@ -797,10 +763,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Cleanup temporary directory. &IDS::cleanup_tmp_directory(); - - # Create new empty file for used rulefiles - # for this provider. - &IDS::write_used_provider_rulefiles_file($provider); } # Perform a reload of the page. @@ -868,7 +830,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { my @enabled_providers = &IDS::get_enabled_providers(); # Write the main providers include file. - &IDS::write_main_used_rulefiles_file(@enabled_providers); + &IDS::write_used_rulefiles_file(@enabled_providers); # Check if oinkmaster has to be executed. if ($oinkmaster eq "True") { @@ -925,7 +887,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &IDS::drop_dl_rulesfile($provider); # Get the name of the provider rulessets include file. - my $provider_used_rulefile = &IDS::get_used_provider_rulesfile_file($provider); + my $provider_used_rulefile = &IDS::get_provider_used_rulesfiles_file($provider); # Drop the file, it is not longer needed. unlink("$provider_used_rulefile"); @@ -947,7 +909,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { my @enabled_providers = &IDS::get_enabled_providers(); # Regenerate main providers include file. - &IDS::write_main_used_rulefiles_file(@enabled_providers); + &IDS::write_used_rulefiles_file(@enabled_providers); # Check if the IDS is running. if(&IDS::ids_is_running()) { From 25652a75d485eaf500a60326373f66e56b902c70 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 10 Apr 2022 11:17:05 +0200 Subject: [PATCH 32/77] ids.cgi: Keep IDS/IPS mode settings when enabling/disabling a provider or autoupdate for it. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 1efead803..3a2e492b9 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -626,7 +626,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { } # Modify the status of the existing entry. - $used_providers{$id} = ["$used_providers{$id}[0]", "$used_providers{$id}[1]", "$status_autoupdate", "$used_providers{$id}[3]"]; + $used_providers{$id} = ["$used_providers{$id}[0]", "$used_providers{$id}[1]", "$status_autoupdate", "$used_providers{$id}[3]", "$used_providers{$id}[4]"]; # Write the changed hash to the providers settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); @@ -821,7 +821,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { } # Modify the status of the existing entry. - $used_providers{$id} = ["$used_providers{$id}[0]", "$used_providers{$id}[1]", "$used_providers{$id}[2]", "$status"]; + $used_providers{$id} = ["$used_providers{$id}[0]", "$used_providers{$id}[1]", "$used_providers{$id}[2]", "$status", "$used_providers{$id}[4]"]; # Write the changed hash to the providers settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); From 1aaa347774a96e54daf26ff0762e63731e94a629 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 10 Apr 2022 11:19:41 +0200 Subject: [PATCH 33/77] ids.cgi: Allow to split working_notice function into two parts. This allows to open the notice and close it at a later time. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 3a2e492b9..56bb5712e 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1830,6 +1830,16 @@ END sub working_notice ($) { my ($message) = @_; + &_open_working_notice ($message); + &_close_working_notice(); +} + +# +## Private function to lock the page and tell the user what is going on. +# +sub _open_working_notice ($) { + my ($message) = @_; + &Header::openpage($Lang::tr{'intrusion detection system'}, 1, ''); &Header::openbigbox('100%', 'left', '', $errormessage); &Header::openbox( 'Waiting', 1,); @@ -1839,8 +1849,15 @@ sub working_notice ($) { $Lang::tr{ $message - END +} + +# +## Private function to close a working notice. +# +sub _close_working_notice () { + print "\n"; + &Header::closebox(); &Header::closebigbox(); &Header::closepage(); From 44d41fd692ea695708c9cd51ecbf1fab2c7a5c28 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 10 Apr 2022 11:23:49 +0200 Subject: [PATCH 34/77] ids.cgi: Add oinkmaster_web () function. This function is used to regenerate the entire ruleset similar to the one from ids-functions, but is enhanced to print additional status messages. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 58 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 56bb5712e..856f7e3d9 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1863,6 +1863,64 @@ sub _close_working_notice () { &Header::closepage(); } +# +## Function which locks the webpage and basically does the same as the +## oinkmaster function, but provides a lot of HTML formated status output. +# +sub oinkmaster_web () { + my @enabled_providers = &IDS::get_enabled_providers(); + + # Lock the webpage and print message. + &_open_working_notice("$Lang::tr{'ids apply ruleset changes'}"); + + # Check if the files in rulesdir have the correct permissions. + &IDS::_check_rulesdir_permissions(); + + print "

\n"; + + # Cleanup the rules directory before filling it with the new rulests. + &_add_to_notice("Remove old rule structures..."); + &IDS::_cleanup_rulesdir(); + + # Loop through the array of enabled providers. + foreach my $provider (@enabled_providers) { + &_add_to_notice("Extracting ruleset for provider: $provider"); + # Call the extractruleset function. + &IDS::extractruleset($provider); + } + + # Call function to process the ruleset and do all modifications. + &_add_to_notice("Adjust rules and add user defined customizations..."); + &IDS::process_ruleset(@enabled_providers); + + # Call function to merge the classification files. + &_add_to_notice("Merging classifications ..."); + &IDS::merge_classifications(@enabled_providers); + + # Call function to merge the sid to message mapping files. + &_add_to_notice("Merging sid to message files ..."); + &IDS::merge_sid_msg(@enabled_providers); + + # Cleanup temporary directory. + &_add_to_notice("Cleanup temporary directory ..."); + &IDS::cleanup_tmp_directory(); + + # Finished - print a notice. + &_add_to_notice("Finished..."); + + # Close the working notice. + &_close_working_notice(); +} + +# +## Tiny private function to add a given message to a notice table. +# +sub _add_to_notice ($) { + my ($message) = @_; + + print "
  • $message
    • \n"; +} + # ## A tiny function to perform a reload of the webpage after one second. # From faa8c62f6377cf6efa2b4edef1bbe77ede248867 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 10 Apr 2022 11:25:36 +0200 Subject: [PATCH 35/77] ids.cgi: Use new oinkmaster_web function instead the silent one from ids-functions. This will print some nice status messages while the page is locked and the IDS rules get regenerated/altered. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 856f7e3d9..9722e6c36 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -403,10 +403,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &IDS::write_used_rulefiles_file(@enabled_providers); # Lock the webpage and print message. - &working_notice("$Lang::tr{'ids apply ruleset changes'}"); - - # Call oinkmaster to alter the ruleset. - &IDS::oinkmaster(); + &oinkmaster_web(); # Check if the IDS is running. if(&IDS::ids_is_running()) { @@ -449,7 +446,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &reload(); } else { # Call subfunction to launch oinkmaster. - &IDS::oinkmaster(); + &oinkmaster_web(); # Check if the IDS is running. if(&IDS::ids_is_running()) { @@ -493,7 +490,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &IDS::write_used_rulefiles_file(@enabled_providers); # Regenerate ruleset. - &IDS::oinkmaster(); + &oinkmaster_web(); # Check if the IDS is running. if(&IDS::ids_is_running()) { @@ -834,11 +831,8 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Check if oinkmaster has to be executed. if ($oinkmaster eq "True") { - # Lock the webpage and print message. - &working_notice("$Lang::tr{'ids apply ruleset changes'}"); - # Launch oinkmaster. - &IDS::oinkmaster(); + &oinkmaster_web(); } # Check if the IDS is running. @@ -903,7 +897,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { } # Regenerate ruleset. - &IDS::oinkmaster(); + &oinkmaster_web(); # Gather all enabled providers. my @enabled_providers = &IDS::get_enabled_providers(); From 149a3291df07c0b1ba0384b83509bb6a62a1eae2 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 11 Apr 2022 05:47:15 +0200 Subject: [PATCH 36/77] ids.cgi: Do not double display a working notice when removing a ruleset provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 3 --- 1 file changed, 3 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 9722e6c36..1ba4b9b53 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -874,9 +874,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Undef the given ID. undef($cgiparams{'ID'}); - # Lock the webpage and print message. - &working_notice("$Lang::tr{'ids apply ruleset changes'}"); - # Drop the stored ruleset file. &IDS::drop_dl_rulesfile($provider); From 990d111d70b7f5276b5ff3b6729773f1066fcee7 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 11 Apr 2022 05:48:17 +0200 Subject: [PATCH 37/77] ids-functions.pl: Add support for Etags. Etags are used to itentify if an ressource has been changed by sending a special request and an Etag value to the server. If the ressource has changed the server will serve the new content otherwise it will return the 304 (Not-Modified) code. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index cf896f340..b81c63b67 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -83,6 +83,10 @@ our $providers_settings_file = "$settingsdir/providers-settings"; # File which stores the configured settings for whitelisted addresses. our $ignored_file = "$settingsdir/ignored"; +# File which stores HTTP Etags for providers which supports them +# for cache management. +our $etags_file = "$settingsdir/etags"; + # Location where the downloaded rulesets are stored. our $dl_rules_path = "/var/tmp"; @@ -394,6 +398,20 @@ sub downloadruleset ($) { $request->header( 'If-Modified-Since' => "$http_date" ); } + # Read-in Etags file for known Etags if the file is present. + my %etags = (); + &General::readhash("$etags_file", \%etags) if (-f $etags_file); + + # Check if an Etag for the current provider is stored. + if ($etags{$provider}) { + # Grab the stored tag. + my $etag = $etags{$provider}; + + # Add an "If-None-Match header to the request to ask the server if the + # file has been modified. + $request->header( 'If-None-Match' => $etag ); + } + my $dl_attempt = 1; my $response; @@ -438,6 +456,15 @@ sub downloadruleset ($) { # Get the remote size of the downloaded file. my $remote_filesize = $headers->content_length; + # Grab the Etag from response it the server provides one. + if ($response->header('Etag')) { + # Add the Etag to the etags hash. + $etags{$provider} = $response->header('Etag'); + + # Write the etags file. + &General::writehash($etags_file, \%etags); + } + # Perform stat on the tmpfile. my $stat = stat($tmpfile); From 39b5adb9404ae1b986e75437c4203752da8e9167 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 11 Apr 2022 05:57:05 +0200 Subject: [PATCH 38/77] update-ids-ruleset: Only regenerate and reload ruleset on at least one successfull update. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index 52df22a76..c62bdcb3b 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -32,6 +32,9 @@ use Sys::Syslog qw(:DEFAULT setlogsock); # Variable to store if the process has written a lockfile. my $locked; +# Array to store the updated providers. +my @updated_providers = (); + # Hash to store the configured providers. my %providers = (); @@ -135,19 +138,25 @@ foreach my $id (keys %providers) { # Set correct ownership for the downloaded tarball. &IDS::set_ownership("$stored_file"); + + # Add the provider handle to the array of updated providers. + push(@updated_providers, $provider); } } -# Call oinkmaster to alter the ruleset. -&IDS::oinkmaster(); +# Check if at least one provider has been updated successfully. +if (@updated_providers) { + # Call oinkmaster to alter the ruleset. + &IDS::oinkmaster(); -# Set correct ownership for the rulesdir and files. -&IDS::set_ownership("$IDS::rulespath"); + # Set correct ownership for the rulesdir and files. + &IDS::set_ownership("$IDS::rulespath"); -# Check if the IDS is running. -if(&IDS::ids_is_running()) { - # Call suricatactrl to perform a reload. - &IDS::call_suricatactrl("reload"); + # Check if the IDS is running. + if(&IDS::ids_is_running()) { + # Call suricatactrl to perform a reload. + &IDS::call_suricatactrl("reload"); + } } # From 2f154264a02a560b0ef4ff6777833330a110f2a4 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 14 Apr 2022 05:16:25 +0200 Subject: [PATCH 39/77] ids.cg: Regeneate ruleset if the ruleset action (mode) of a provider get changed. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 1ba4b9b53..83928b258 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -642,6 +642,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { my $subscription_code = $cgiparams{'SUBSCRIPTION_CODE'}; my $status_autoupdate; my $mode; + my $regenerate_ruleset_required; # Handle autoupdate checkbox. if ($cgiparams{'ENABLE_AUTOUPDATE'} eq "on") { @@ -697,6 +698,15 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Undef the given ID. undef($cgiparams{'ID'}); + # Grab the configured mode. + my $stored_mode = $used_providers{$id}[4]; + + # Check if the ruleset action (mode) has been changed. + if ($stored_mode ne $mode) { + # It has been changed, so the ruleset needs to be regenerated. + $regenerate_ruleset_required = "1"; + } + # Grab the configured status of the corresponding entry. $status = $used_providers{$id}[3]; } else { @@ -770,6 +780,14 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { } } + # Check if the ruleset has to be regenerated. + if ($regenerate_ruleset_required) { + # Call oinkmaster web function. + &oinkmaster_web(); + + # Perform a reload of the page. + &reload(); + } } # Undefine providers flag. From c00609ce56cab337d352e69599144683192dec8f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 14 Apr 2022 05:47:55 +0200 Subject: [PATCH 40/77] convert-ids-backend-files: Successor of the convert-ids-modifications-files converter. This converter also will convert the used rulesfiles file for the providers. Signed-off-by: Stefan Schantl --- ...cation-files => convert-ids-backend-files} | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) rename config/suricata/{convert-ids-modification-files => convert-ids-backend-files} (71%) diff --git a/config/suricata/convert-ids-modification-files b/config/suricata/convert-ids-backend-files similarity index 71% rename from config/suricata/convert-ids-modification-files rename to config/suricata/convert-ids-backend-files index 555deaf18..78e8edbf1 100644 --- a/config/suricata/convert-ids-modification-files +++ b/config/suricata/convert-ids-backend-files @@ -30,6 +30,56 @@ exit 0 unless (-f "$IDS::settingsdir/oinkmaster.conf"); # Get all supported providers. my @providers = &IDS::get_ruleset_providers(); +# +## Step 1: Convert used rules files. +# + +# Loop through the array of known providers. +foreach my $provider (@providers) { + my %used_rulesfiles = (); + + # Generate old filename which contained the used rulesfile. + my $old_used_rulesfiles_file = "$IDS::settingsdir/suricata-$provider\-used-rulefiles.yaml"; + + # Skip the provider if there is no used rulesfiles file available. + next unless (-f $old_used_rulesfiles_file); + + # Open the used rulesfiles file. + open(FILE, "$old_used_rulesfiles_file"); + + # Read-in the file content. + my @file = ; + + # Close file handle. + close(FILE); + + # Loop through the file content. + foreach my $line(@file) { + chomp($line); + + # Grab the used rulesfile name from the line. + if ($line =~ /^\s-\s(.*)/) { + my $rulesfile = $1; + + # Add the used rulesfile to the has of used rulesfile for this provider. + $used_rulesfiles{$rulesfile} = "enabled"; + } + } + + # Get the filename for the new used rulesfiles file. + my $used_rulesfiles_file = &IDS::get_provider_used_rulesfiles_file($provider); + + # Write the file. + &General::writehash("$used_rulesfiles_file", \%used_rulesfiles); + + # Set the correct ownership for the new file. + &IDS::set_ownership("$used_rulesfiles_file"); +} + +# +## Step 2: Convert ruleset modifictaion files. +# + # Loop through the array of providers. foreach my $provider (@providers) { my %modifications = (); From 9f7702544abc0a906d14ccdcf0e4b03239a8fc33 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:10:45 +0200 Subject: [PATCH 41/77] convert-ids-backend-files: Regenerate ruleset and used rulesets file. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 78e8edbf1..e7cef039d 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -128,3 +128,20 @@ foreach my $provider (@providers) { # Set correct ownership for the new modifications file. &IDS::set_ownership("$new_modifications_file"); } + +# +## Step 3: Regenerate the ruleset. +# + +# Call oinkmaster wrapper function. +&IDS::oinkmaster(); + +# +## Step 4: Write new config file for suricata which contains the used rulesfiles. +# + +# Get enabled providers. +my @enabled_providers = &IDS::get_enabled_providers(); + +# Write used rulesfiles file. +&IDS::write_used_rulefiles_file(@enabled_providers); From 70b1672d94f3f6c3cfe82bf65df65125df0b0014 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:12:56 +0200 Subject: [PATCH 42/77] convert-ids-backend-files: Remove converted files. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index e7cef039d..12849929b 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -74,6 +74,9 @@ foreach my $provider (@providers) { # Set the correct ownership for the new file. &IDS::set_ownership("$used_rulesfiles_file"); + + # Delete old used rulesfiles file. + unlink("$old_used_rulesfiles_file"); } # @@ -127,6 +130,9 @@ foreach my $provider (@providers) { # Set correct ownership for the new modifications file. &IDS::set_ownership("$new_modifications_file"); + + # Delete old modifications file. + unlink("$old_modifications_file"); } # From c215cfd8873130362b0665696e06a79279f79abd Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:13:23 +0200 Subject: [PATCH 43/77] convert-ids-backend-files: Remove old backend related files. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 12849929b..a982b323b 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -27,6 +27,22 @@ require '/var/ipfire/ids-functions.pl'; # Exit if there is no main oinkmaster config file anymore. exit 0 unless (-f "$IDS::settingsdir/oinkmaster.conf"); +# Array of old files, which are safe to drop. +my @files_to_drop = ( + # Old settings files of oinkmaster. + "$IDS::settingsdir/oinkmaster.conf", + "$IDS::settingsdir/oinkmaster-disabled-sids.conf", + "$IDS::settingsdir/oinkmaster-enabled-sids.conf", + "$IDS::settingsdir/oinkmaster-modify-sids.conf", + "$IDS::settingddir/oinkmaster-provider-includes.conf", + + # Old settingsfiles for suricata. + "$IDS::settingsdir/suricata-default-rules.yaml", + "$IDS::settingsdir/suricata-static-included-rulefiles.yaml", + "$IDS::settingsdir/suricata-used-providers.yaml", + "$IDS::settingsdir/suricata-used-rulefiles.yaml" +); + # Get all supported providers. my @providers = &IDS::get_ruleset_providers(); @@ -151,3 +167,13 @@ my @enabled_providers = &IDS::get_enabled_providers(); # Write used rulesfiles file. &IDS::write_used_rulefiles_file(@enabled_providers); + +# +## Step 5: Remove unneeded orphaned files. +# + +# Loop through the array of files which are safe to drop. +foreach my $file (@files_to_drop) { + # Remove the file if it exists. + unlink("$file") if (-f "$file"); +} From 4f513522feeb88a447a861d414eead6432ce784f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:18:37 +0200 Subject: [PATCH 44/77] ids-functions.pl: Do not use a hard-code temporary download location. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index b81c63b67..ef79199b3 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -130,6 +130,9 @@ my $suricatactrl = "/usr/local/bin/suricatactrl"; # Prefix for each downloaded ruleset. my $dl_rulesfile_prefix = "idsrules"; +# Temporary directory to download the rules files. +my $tmp_dl_directory = "/var/tmp"; + # Temporary directory where the rulesets will be extracted. my $tmp_directory = "/tmp/ids_tmp"; @@ -371,9 +374,9 @@ sub downloadruleset ($) { # Pass the requested URL to the downloader. my $request = HTTP::Request->new(GET => $url); - # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". + # Generate temporary file name, located in the tempoary download directory and with a suffix of ".tmp". # The downloaded file will be stored there until some sanity checks are performed. - my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); + my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "$tmp_dl_directory/", UNLINK => 0 ); my $tmpfile = $tmp->filename(); # Call function to get the final path and filename for the downloaded file. From b570d35c0aff4c1d126be539bbb009830a1fbb7f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:19:20 +0200 Subject: [PATCH 45/77] ids-functions.pl: Change location for downloaded rulesfiles to "/var/cache/suricata/". Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index ef79199b3..b433ee90f 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -88,7 +88,7 @@ our $ignored_file = "$settingsdir/ignored"; our $etags_file = "$settingsdir/etags"; # Location where the downloaded rulesets are stored. -our $dl_rules_path = "/var/tmp"; +our $dl_rules_path = "/var/cache/suricata"; # File to store any errors, which also will be read and displayed by the wui. our $storederrorfile = "/tmp/ids_storederror"; From c2eac6fcd4281834409700066b25061d15ca0d6c Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:52:01 +0200 Subject: [PATCH 46/77] convert-ids-backend-files: Move already downloaded files to new location. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 42 ++++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index a982b323b..0b81b008e 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -21,6 +21,8 @@ use strict; +use File::Copy; + require '/var/ipfire/general-functions.pl'; require '/var/ipfire/ids-functions.pl'; @@ -43,11 +45,41 @@ my @files_to_drop = ( "$IDS::settingsdir/suricata-used-rulefiles.yaml" ); +# +# Step 1: Move downloaded files to new location. +# + +my $old_dl_rulesfiles_dir = "/var/tmp"; + +# Open old rules directory and do a directory listsing. +opendir(DIR, "$old_dl_rulesfiles_dir"); + +# Loop through the files of the directory. +while (my $file = readdir(DIR)) { + # Check if the file starts with an "idsrules-". + if ($file =~ /^idsrules-/) { + # Grab the mtime of the file. + my $mtime=(stat "$old_dl_rulesfiles_dir/$file")[9]; + + # Move the file to its new location. + move("$old_dl_rulesfiles_dir/$file", "$IDS::dl_rules_path/$file"); + + # Set correct ownership. + &IDS::set_ownership("$IDS::dl_rules_path/$file"); + + # Restore the mtime on the file. + utime(time(), "$mtime", "$IDS::dl_rules_path/$file"); + } +} + +# Close directory handle. +closedir(DIR); + # Get all supported providers. my @providers = &IDS::get_ruleset_providers(); # -## Step 1: Convert used rules files. +## Step 2: Convert used rules files. # # Loop through the array of known providers. @@ -96,7 +128,7 @@ foreach my $provider (@providers) { } # -## Step 2: Convert ruleset modifictaion files. +## Step 3: Convert ruleset modifictaion files. # # Loop through the array of providers. @@ -152,14 +184,14 @@ foreach my $provider (@providers) { } # -## Step 3: Regenerate the ruleset. +## Step 4: Regenerate the ruleset. # # Call oinkmaster wrapper function. &IDS::oinkmaster(); # -## Step 4: Write new config file for suricata which contains the used rulesfiles. +## Step 5: Write new config file for suricata which contains the used rulesfiles. # # Get enabled providers. @@ -169,7 +201,7 @@ my @enabled_providers = &IDS::get_enabled_providers(); &IDS::write_used_rulefiles_file(@enabled_providers); # -## Step 5: Remove unneeded orphaned files. +## Step 6: Remove unneeded orphaned files. # # Loop through the array of files which are safe to drop. From a15c9b16b404bc1970fd016104560e8fd24b5edb Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 05:59:33 +0200 Subject: [PATCH 47/77] IDS: Move autoupdate logic to cron. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 37 ++++++--------------------------- config/cron/crontab | 3 +++ html/cgi-bin/ids.cgi | 26 ----------------------- 3 files changed, 9 insertions(+), 57 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index b433ee90f..8e1137ce0 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -143,10 +143,7 @@ my $tmp_rules_directory = "$tmp_directory/rules"; my $tmp_conf_directory = "$tmp_directory/conf"; # Array with allowed commands of suricatactrl. -my @suricatactrl_cmds = ( 'start', 'stop', 'restart', 'reload', 'fix-rules-dir', 'cron' ); - -# Array with supported cron intervals. -my @cron_intervals = ('off', 'daily', 'weekly' ); +my @suricatactrl_cmds = ( 'start', 'stop', 'restart', 'reload', 'fix-rules-dir' ); # Array which contains the HTTP ports, which statically will be declared as HTTP_PORTS in the # http_ports_file. @@ -1172,34 +1169,12 @@ sub call_suricatactrl ($) { # Skip current command unless the given one has been found. next unless($cmd eq $option); - # Check if the given command is "cron". - if ($option eq "cron") { - # Check if an interval has been given. - if ($interval) { - # Check if the given interval is valid. - foreach my $element (@cron_intervals) { - # Skip current element until the given one has been found. - next unless($element eq $interval); + # Call the suricatactrl binary and pass the requrested + # option to it. + &General::system("$suricatactrl", "$option"); - # Call the suricatactrl binary and pass the "cron" command - # with the requrested interval. - &General::system("$suricatactrl", "$option", "$interval"); - - # Return "1" - True. - return 1; - } - } - - # If we got here, the given interval is not supported or none has been given. - Return nothing. - return; - } else { - # Call the suricatactrl binary and pass the requrested - # option to it. - &General::system("$suricatactrl", "$option"); - - # Return "1" - True. - return 1; - } + # Return "1" - True. + return 1; } # Command not found - return nothing. diff --git a/config/cron/crontab b/config/cron/crontab index d94dd65c2..d61d26619 100644 --- a/config/cron/crontab +++ b/config/cron/crontab @@ -62,6 +62,9 @@ HOME=/ # Update location database %hourly,random * [ -f "/var/ipfire/red/active" ] && /usr/local/bin/update-location-database >/dev/null 2>&1 +# Update surciata rules. +%daily,random * [ -f "/var/ipfire/red/active" ] && /usr/local/bin/update-ids-ruleset >/dev/null 2>&1 + # Retry sending spooled mails regularly %hourly * /usr/sbin/dma -q diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 83928b258..053520b57 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -561,12 +561,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &General::writehash("$IDS::ids_settings_file", \%cgiparams); } - # Check if the the automatic rule update hass been touched. - if($cgiparams{'AUTOUPDATE_INTERVAL'} ne $oldidssettings{'AUTOUPDATE_INTERVAL'}) { - # Call suricatactrl to set the new interval. - &IDS::call_suricatactrl("cron", $cgiparams{'AUTOUPDATE_INTERVAL'}); - } - # Generate file to store the home net. &IDS::generate_home_net_file(); @@ -981,22 +975,12 @@ sub show_mainpage() { &General::readhash("$IDS::ids_settings_file", \%idssettings); &General::readhasharray("$IDS::providers_settings_file", \%used_providers); - # If no autoupdate intervall has been configured yet, set default value. - unless(exists($idssettings{'AUTOUPDATE_INTERVAL'})) { - # Set default to "weekly". - $idssettings{'AUTOUPDATE_INTERVAL'} = 'weekly'; - } - # Read-in ignored hosts. &General::readhasharray("$IDS::ignored_file", \%ignored) if (-e $IDS::ignored_file); $checked{'ENABLE_IDS'}{'off'} = ''; $checked{'ENABLE_IDS'}{'on'} = ''; $checked{'ENABLE_IDS'}{$idssettings{'ENABLE_IDS'}} = "checked='checked'"; - $selected{'AUTOUPDATE_INTERVAL'}{'off'} = ''; - $selected{'AUTOUPDATE_INTERVAL'}{'daily'} = ''; - $selected{'AUTOUPDATE_INTERVAL'}{'weekly'} = ''; - $selected{'AUTOUPDATE_INTERVAL'}{$idssettings{'AUTOUPDATE_INTERVAL'}} = "selected='selected'"; # Draw current state of the IDS &Header::openbox('100%', 'left', $Lang::tr{'intrusion detection system'}); @@ -1121,16 +1105,6 @@ print < $Lang::tr{'ids automatic rules update'} - - - - - -

    From a2c56ead7367995ff743cc5c75aec8c4fb195f83 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 15 Apr 2022 06:02:49 +0200 Subject: [PATCH 48/77] ids-functions.pl: Remove read_enabled_disabled_sids_file() function. Not longer needed and therefore dead code. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 51 --------------------------------- 1 file changed, 51 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 8e1137ce0..c916926de 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1077,57 +1077,6 @@ sub drop_dl_rulesfile ($) { } } -# -## Function to read-in the given enabled or disables sids file. -# -sub read_enabled_disabled_sids_file($) { - my ($file) = @_; - - # Temporary hash to store the sids and their state. It will be - # returned at the end of this function. - my %temphash; - - # Open the given filename. - open(FILE, "$file") or die "Could not open $file. $!\n"; - - # Loop through the file. - while() { - # Remove newlines. - chomp $_; - - # Skip blank lines. - next if ($_ =~ /^\s*$/); - - # Skip coments. - next if ($_ =~ /^\#/); - - # Splitt line into sid and state part. - my ($state, $sid) = split(" ", $_); - - # Skip line if the sid is not numeric. - next unless ($sid =~ /\d+/ ); - - # Check if the sid was enabled. - if ($state eq "enablesid") { - # Add the sid and its state as enabled to the temporary hash. - $temphash{$sid} = "enabled"; - # Check if the sid was disabled. - } elsif ($state eq "disablesid") { - # Add the sid and its state as disabled to the temporary hash. - $temphash{$sid} = "disabled"; - # Invalid state - skip the current sid and state. - } else { - next; - } - } - - # Close filehandle. - close(FILE); - - # Return the hash. - return %temphash; -} - # ## Function to check if the IDS is running. # From 2e558477da7438d2bd79411279ae1502f044c787 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 14:39:09 +0200 Subject: [PATCH 49/77] convert-ids-backend-files: Convert MONITOR_TRAFFIC_ONLY settings. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 40 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 0b81b008e..302ed5232 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -184,14 +184,48 @@ foreach my $provider (@providers) { } # -## Step 4: Regenerate the ruleset. +## Step 4: Convert MONTIOR_TRAFFIC_ONLY setting. +# + +my %ids_settings = (); +my %provider_settings = (); + +&General::readhash("$IDS::ids_settings_file", \%ids_settings); +&General::readhasharray("$IDS::providers_settings_file", \%provider_settings); + +# Default to IPS mode. +my $mode = "IPS"; + +# Check if MONTOR_TRAFFIC_ONLY has been activated. +if(($ids_settings{'MONITOR_TRAFFIC_ONLY'} && $ids_settings{'MONITOR_TRAFFIC_ONLY'} eq "on")) { + $mode = "IDS"; +} + +# Loop through the hash of providers. +foreach my $key (keys %provider_settings) { + # Get and dereference settings array from hash. + my @settings = @{ $provider_settings{$key} }; + + # Add the mode as last element to the settings array. + push(@settings, $mode); + + # Assign the new settings to the hash. + $provider_settings{$key} = [ @settings ]; +} + +# Write back providers settings. +&General::writehasharray("$IDS::providers_settings_file", \%provider_settings); + +# +## Step 5: Regenerate the ruleset. +# # # Call oinkmaster wrapper function. &IDS::oinkmaster(); # -## Step 5: Write new config file for suricata which contains the used rulesfiles. +## Step 6: Write new config file for suricata which contains the used rulesfiles. # # Get enabled providers. @@ -201,7 +235,7 @@ my @enabled_providers = &IDS::get_enabled_providers(); &IDS::write_used_rulefiles_file(@enabled_providers); # -## Step 6: Remove unneeded orphaned files. +## Step 7: Remove unneeded orphaned files. # # Loop through the array of files which are safe to drop. From b75baeff28412bec16dd72e4251d24c371c3fd5d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 14:42:22 +0200 Subject: [PATCH 50/77] suricata: Do not longer install YAML file for default rules. This file got obsolete, because it's content will be generated dynamically by the backend code. Signed-off-by: Stefan Schantl --- config/suricata/suricata-default-rules.yaml | 18 ------------------ lfs/suricata | 6 ------ 2 files changed, 24 deletions(-) delete mode 100644 config/suricata/suricata-default-rules.yaml diff --git a/config/suricata/suricata-default-rules.yaml b/config/suricata/suricata-default-rules.yaml deleted file mode 100644 index d6c358add..000000000 --- a/config/suricata/suricata-default-rules.yaml +++ /dev/null @@ -1,18 +0,0 @@ -%YAML 1.1 ---- - -# Default rules which helps - - /usr/share/suricata/rules/app-layer-events.rules - - /usr/share/suricata/rules/decoder-events.rules - - /usr/share/suricata/rules/dhcp-events.rules - - /usr/share/suricata/rules/dns-events.rules - - /usr/share/suricata/rules/files.rules - - /usr/share/suricata/rules/http-events.rules - - /usr/share/suricata/rules/ipsec-events.rules - - /usr/share/suricata/rules/kerberos-events.rules - - /usr/share/suricata/rules/nfs-events.rules - - /usr/share/suricata/rules/ntp-events.rules - - /usr/share/suricata/rules/smb-events.rules - - /usr/share/suricata/rules/smtp-events.rules - - /usr/share/suricata/rules/stream-events.rules - - /usr/share/suricata/rules/tls-events.rules diff --git a/lfs/suricata b/lfs/suricata index 65f5e504c..819f562a3 100644 --- a/lfs/suricata +++ b/lfs/suricata @@ -98,12 +98,6 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) # Install IPFire related config file. install -m 0644 $(DIR_SRC)/config/suricata/suricata.yaml /etc/suricata - # Install yaml file for loading default rules. - install -m 0664 $(DIR_SRC)/config/suricata/suricata-default-rules.yaml /var/ipfire/suricata - - # Set correct ownership for the default rules file. - chown nobody:nobody /var/ipfire/suricata/suricata-default-rules.yaml - # Create emtpy rules directory. -mkdir -p /var/lib/suricata From d44d4ccf34132b77c8cf3d4ace7eab99a4717a53 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 14:48:35 +0200 Subject: [PATCH 51/77] suricata: Create directory to store the downloaded ruleset files. Signed-off-by: Stefan Schantl --- lfs/suricata | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lfs/suricata b/lfs/suricata index 819f562a3..9a8583ea9 100644 --- a/lfs/suricata +++ b/lfs/suricata @@ -101,6 +101,9 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) # Create emtpy rules directory. -mkdir -p /var/lib/suricata + # Create empty cache directory. + -mkdir -p /var/cache/suricata + # Move config files for references, threshold and classification # to the rules directory. rm -rfv /etc/suricata/*.config @@ -113,6 +116,9 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) # contained files chown -R nobody:nobody /var/lib/suricata + # Set correct ownership for the cache directory. + chown nobody:nobody /var/cache/suricata + # Create logging directory. -mkdir -p /var/log/suricata From 0d99255c0614d0218912724b97f6cfdb4811a895 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 14:49:52 +0200 Subject: [PATCH 52/77] suricata: Create empty threshold.config file. The file is referenced in the suricata config file and if not present some ugly warnings will be displayed/logged during startup. Signed-off-by: Stefan Schantl --- lfs/suricata | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lfs/suricata b/lfs/suricata index 9a8583ea9..d09f87a5c 100644 --- a/lfs/suricata +++ b/lfs/suricata @@ -112,6 +112,12 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) # (File has to be writeable for the nobody user) chown nobody:nobody /usr/share/suricata/classification.config + # Create empty threshold config file. + touch /usr/share/suricata/threshold.config + + # Set correct ownership for the threshold.config file. + chown nobody:nobody /usr/share/suricata/threshold.config + # Set correct ownership for /var/lib/suricata and the # contained files chown -R nobody:nobody /var/lib/suricata From d2bf4d377f698076e53a56a9784a0b70d8ed3388 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 14:51:48 +0200 Subject: [PATCH 53/77] suricata: Rootfile update. Signed-off-by: Stefan Schantl --- config/rootfiles/common/suricata | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/rootfiles/common/suricata b/config/rootfiles/common/suricata index 7f9ff8156..dcb6fe91d 100644 --- a/config/rootfiles/common/suricata +++ b/config/rootfiles/common/suricata @@ -20,6 +20,7 @@ usr/bin/suricata usr/share/suricata #usr/share/suricata/classification.config #usr/share/suricata/reference.config +#usr/share/suricata/threshold.config #usr/share/suricata/rules #usr/share/suricata/rules/app-layer-events.rules #usr/share/suricata/rules/decoder-events.rules @@ -37,7 +38,7 @@ usr/share/suricata #usr/share/suricata/rules/smtp-events.rules #usr/share/suricata/rules/stream-events.rules #usr/share/suricata/rules/tls-events.rules -var/ipfire/suricata/suricata-default-rules.yaml +var/cache/suricata var/lib/suricata var/log/suricata #var/log/suricata/certs From 93af000b8b3f86008040cb5a62405b158c270fe7 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 14:54:11 +0200 Subject: [PATCH 54/77] oinkmaster: Drop package. Signed-off-by: Stefan Schantl --- config/oinkmaster/oinkmaster.conf | 429 ----------------------------- config/rootfiles/common/oinkmaster | 2 - lfs/oinkmaster | 79 ------ make.sh | 1 - 4 files changed, 511 deletions(-) delete mode 100644 config/oinkmaster/oinkmaster.conf delete mode 100644 config/rootfiles/common/oinkmaster delete mode 100644 lfs/oinkmaster diff --git a/config/oinkmaster/oinkmaster.conf b/config/oinkmaster/oinkmaster.conf deleted file mode 100644 index 4d4ee40ef..000000000 --- a/config/oinkmaster/oinkmaster.conf +++ /dev/null @@ -1,429 +0,0 @@ -# $Id: oinkmaster.conf,v 1.132 2006/02/02 12:05:08 andreas_o Exp $ # - -# This file is pretty big by default, but don't worry. -# The only things required are "path" and "update_files". You must also -# set "url" to point to the correct rules archive for your version of -# Snort, unless you prefer to specify this on the command line. -# The rest in here is just a few recommended defaults, and examples -# how to use all the other optional features and give some ideas how they -# could be used. - -# Remember not to let untrusted users edit Oinkmaster configuration -# files, as things like the PATH to use during execution is defined -# in here. - - -# Use "url = " to specify the location of the rules archive to -# download. The url must begin with http://, https://, ftp://, file:// -# or scp:// and end with .tar.gz or .tgz, and the file must be a -# gzipped tarball what contains a directory named "rules". -# You can also point to a local directory with dir://. -# Multiple "url = " lines can be specified to grab multiple rules -# archives from different locations. -# -# Note: if URL is specified on the command line, it overrides all -# possible URLs specified in the configuration file(s). -# -# The location of the official Snort rules you should use depends -# on which Snort version you run. Basically, you should go to -# http://www.snort.org/rules/ and follow the instructions -# there to pick the right URL for your version of Snort -# (and remember to update the URL when upgrading Snort in the -# future). You can of course also specify locations to third party -# rules. -# -# As of March 2005, you must register on the Snort site to get access -# to the official Snort rules. This will get you an "oinkcode". -# You then specify the URL as -# http://www.snort.org/pub-bin/oinkmaster.cgi// -# For example, if your code is 5a081649c06a277e1022e1284b and -# you use Snort 2.4, the url to use would be (without the wrap): -# http://www.snort.org/pub-bin/oinkmaster.cgi/ -# 5a081649c06a277e1022e1284bdc8fabda70e2a4/snortrules-snapshot-2.4.tar.gz -# See the Oinkmaster FAQ Q1 and http://www.snort.org/rules/ for -# more information. - - -# URL examples follows. Replace with the code you get on the -# Snort site in your registered user profile. - -# Example for Snort 2.4 -# url = http://www.snort.org/pub-bin/oinkmaster.cgi//snortrules-snapshot-2.4.tar.gz -# url = http://www.snort.org/pub-bin/oinkmaster.cgi//snortrules-snapshot-2.4.tar.gz - -# Example for Snort-current ("current" means cvs snapshots). -#url = http://www.snort.org/pub-bin/oinkmaster.cgi//snortrules-snapshot-CURRENT.tar.gz - -# Example for Community rules -# url = http://www.snort.org/pub-bin/downloads.cgi/Download/comm_rules/Community-Rules.tar.gz - -# Example for rules from the Bleeding Snort project -# url = http://www.bleedingsnort.com/bleeding.rules.tar.gz - -# If you prefer to download the rules archive from outside Oinkmaster, -# you can then point to the file on your local filesystem by using -# file://, for example: -# url = file:///tmp/snortrules.tar.gz - -# In rare cases you may want to grab the rules directly from a -# local directory (don't confuse this with the output directory). -# url = dir:///etc/snort/src/rules - -# Example to use scp to copy the rules archive from another host. -# Only OpenSSH is tested. See the FAQ for more information. -# url = scp://user@somehost.example.com:/somedir/snortrules.tar.gz - -# If you use -u scp://... and need to specify a private ssh key (passed -# as -i to the scp command) you can specify it here or add an -# entry in ~/.ssh/config for the Oinkmaster user as described in the -# OpenSSH manual. -# scp_key = /home/oinkmaster/oinkmaster_privkey - - -# The PATH to use during execution. If you prefer to use external -# binaries (i.e. use_external_bins=1, see below), tar and gzip must be -# found, and also wget if downloading via ftp, http or https. All with -# optional .exe suffix. If you're on Cygwin, make sure that the path -# contains the Cygwin binaries and not the native Win32 binaries or -# you will get problems. -# Assume UNIX style by default: -path = /bin:/usr/bin:/usr/local/bin - -# Example if running native Win32 or standalone Cygwin: -# path = c:\oinkmaster;c:\oinkmaster\bin - -# Example if running standalone Cygwin and you prefer Cygwin style path: -# path = /cygdrive/c/oinkmaster:/cygdrive/c/oinkmaster/bin - - -# We normally use external binaries (wget, tar and gzip) since they're -# already available on most systems and do a good job. If you have the -# Perl modules Archive::Tar, IO::Zlib and LWP::UserAgent, you can use -# those instead if you like. You can set use_external_bins below to -# choose which method you prefer. It's set to 0 by default on Win32 -# (i.e. use Perl modules), and 1 on other systems (i.e. use external -# binaries). The reason for that is that the required Perl modules -# are included on Windows/ActivePerl 5.8.1+, so it's easier to use -# those than to install the ported Unix tools. (Note that if you're -# using scp to download the archive, external scp binary is still -# used.) -# use_external_bins = 0 - - -# Temporary directory to use. This directory must exist when starting and -# Oinkmaster will then create a temporary sub directory in here. -# Keep it as a #comment if you want to use the default. -# The default will be checked for in the environment variables TMP, -# TMPDIR or TEMPDIR, or otherwise use "/tmp" if none of them was set. - -# Example for UNIX. -# tmpdir = /home/oinkmaster/tmp/ - -# Example if running native Win32 or Cygwin. -# tmpdir = c:\tmp - -# Example if running Cygwin and you prefer Cygwin style path. -# tmpdir = /cygdrive/c/tmp - - -# The umask to use during execution if you want it to be something -# else than the current value when starting Oinkmaster. -# This will affect the mode bits when writing new files. -# Keep it commented out to keep your system's current umask. -# umask = 0027 - - -# Files in the archive(s) matching this regular expression will be -# checked for changes, and then updated or added if needed. -# All other files will be ignored. You can then choose to skip -# individual files by specifying the "skipfile" keyword below. -# Normally you shouldn't need to change this one. -update_files = \.rules$|\.config$|\.conf$|\.txt$|\.map$ - - -# Regexp of keywords that starts a Snort rule. -# May be useful if you create your own ruletypes and want those -# lines to be regarded as rules as well. -# rule_actions = alert|drop|log|pass|reject|sdrop|activate|dynamic - - -# If the number of rules files in the downloaded archive matching the -# 'update_files' regexp is below min_files, or if the number -# of rules is below min_rules, the rules are regarded as broken -# and the update is aborted with an error message. -# Both are set to 1 by default (i.e. the archive is only regarded as -# broken if it's totally empty). -# If you download from multiple URLs, the count is the total number -# of files/rules across all archives. -# min_files = 1 -# min_rules = 1 - - -# By default, a basic sanity check is performed on most paths/filenames -# to see if they contain illegal characters that may screw things up. -# If this check is too strict for your system (e.g. you get bogus -# "illegal characters in filename" errors because of your local language -# etc) and you're sure you want to disable the checks completely, -# set use_path_checks to 0. -# use_path_checks = 1 - - -# If you want Oinkmaster to send a User-Agent HTTP header string -# other than the default one for wget/LWP, set this variable. -# user_agent = Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) - - -# You can include other files anywhere in here by using -# "include ". will be parsed just like a regular -# oinkmaster.conf as soon as the include statement is seen, and then -# return and continue parsing the rest of the original file. If an -# option is redefined, it will override the previous value. You can use -# as many "include" statements as you wish, and also include even more -# files from included files. Example to load stuff from "/etc/foo.conf". -# include /etc/foo.conf - -# Include file for provider specific includes. -include /var/ipfire/suricata/oinkmaster-provider-includes.conf - -# Include file which defines the runmode of suricata. -include /var/ipfire/suricata/oinkmaster-modify-sids.conf - -####################################################################### -# Files to totally skip (i.e. never update or check for changes) # -# # -# Syntax: skipfile filename # -# or: skipfile filename1, filename2, filename3, ... # -####################################################################### - -# Ignore local.rules from the rules archive by default since we might -# have put some local rules in our own local.rules and we don't want it -# to get overwritten by the empty one from the archive after each -# update. -skipfile local.rules - -# The file deleted.rules contains rules that have been deleted from -# other files, so there is usually no point in updating it. -skipfile deleted.rules - -# Also skip snort.conf by default since we don't want to overwrite our -# own snort.conf if we have it in the same directory as the rules. If -# you have your own production copy of snort.conf in another directory, -# it may be really nice to check for changes in this file though, -# especially since variables are sometimes added or modified and -# new/old files are included/excluded. -#skipfile snort.conf - -# You may want to consider ignoring threshold.conf for the same reasons -# as for snort.conf, i.e. if you customize it locally and don't want it -# to become overwritten by the default one. It may be better to put -# local thresholding/suppressing in some local file and still update -# and use the official one though, in case important stuff is added to -# it some day. We do update it by default, but it's your call. -skipfile threshold.conf - -# If you update from multiple URLs at the same time you may need to -# ignore the sid-msg.map (and generate it yourself if you need one) as -# it's usually included in each rules tarball. See the FAQ for more info. -# skipfile sid-msg.map - - - -########################################################################## -# SIDs to modify after each update (only for the skilled/stupid/brave). # -# Don't use it unless you have to. There is nothing that stops you from # -# modifying rules in such ways that they become invalid or generally # -# break things. You have been warned. # -# If you just want to disable SIDs, please skip this section and have a # -# look at the "disablesid" keyword below. # -# # -# You may specify multiple modifysid directives for the same SID (they # -# will be processed in order of appearance), and you may also specify a # -# list of SIDs on which the substitution should be applied. # -# If the argument is in the form something.something it's regarded # -# as a filename and the substitution will apply on all rules in that # -# file. The wildcard ("*") can be used to apply the substitution on all # -# rules regardless of the SID or file. Please avoid using #comments # -# at the end of modifysid lines, they may confuse the parser in some # -# situations. # -# # -# Syntax: # -# modifysid SID "replacethis" | "withthis" # -# or: # -# modifysid SID1, SID2, SID3, ... "replacethis" | "withthis" # -# or: # -# modifysid file "replacethis" | "withthis" # -# or: # -# modifysid * "replacethis" | "withthis" # -# # -# The strings within the quotes will basically be passed to a # -# s/replacethis/withthis/ statement in Perl, so they must be valid # -# regular expressions. The strings are case-insensitive and only the # -# first occurrence will be replaced. If there are multiple occurrences # -# you want to replace, simply repeat the same modifysid line. # -# As the strings are regular expressions, you MUST escape special # -# characters like $ \ / ( ) | by prepending a "\" to them. # -# # -# If you specify a modifysid statement for a multi-line rule, Oinkmaster # -# will first translate the rule into a single-line version and then # -# perform the substitution, so you don't have to care about the trailing # -# backslashes and newlines. # -# # -# If you use backreference variables in the substitution expression, # -# it's strongly recommended to specify them as ${1} instead of $1 and so # -# on, to avoid parsing confusion with unexpected results in some # -# situations. Note that modifysid statements will process both active # -# and inactive (disabled) rules. # -# # -# You may want to check out README.templates and template-examples.conf # -# to find how you can simplify the modifysid usage by using templates. # -########################################################################## - -# Example to enable a rule (in this case SID 1325) that is disabled by -# default, by simply replacing leading "#alert" with "alert". -# (You should really use 'enablesid' for this though.) -# Oinkmaster removes whitespaces next to the leading "#" so you don't -# have to worry about that, but be careful about possible whitespace in -# other places when writing the regexps. -# modifysid 1325 "^#alert" | "alert" - -# You could also do this to enable it no matter what type of rule it is -# (alert, log, pass, etc). -# modifysid 1325 "^#" | "" - -# Example to add "tag" stuff to SID 1325. -# modifysid 1325 "sid:1325;" | "sid:1325; tag: host, src, 300, seconds;" - -# Example to make SID 1378 a 'drop' rule (valid if you're running -# Snort_inline). -# modifysid 1378 "^alert" | "drop" - -# Example to replace first occurrence of $EXTERNAL_NET with $HOME_NET -# in SID 302. -# modifysid 302 "\$EXTERNAL_NET" | "\$HOME_NET" - -# You can also specify that a substitution should apply on multiple SIDs. -# modifysid 302,429,1821 "\$EXTERNAL_NET" | "\$HOME_NET" - -# You can take advantage of the fact that it's regular expressions and -# do more complex stuff. This example (for Snort_inline) adds a 'replace' -# statement to SID 1324 that replaces "/bin/sh" with "/foo/sh". -# modifysid 1324 "(content\s*:\s*"\/bin\/sh"\s*;)" | \ -# "${1} replace:"\/foo\/sh";" - -# If you for some reason would like to add a comment inside the actual -# rules file, like the reason why you disabled this rule, you can do -# like this (you would normally add such comments in oinkmaster.conf -# though). -# modifysid 1324 "(.+)" | "# 20020101: disabled this rule just for fun:\n#${1}" - -# Here is an example that is actually useful. Let's say you don't care -# about incoming welchia pings (detected by SID 483 at the time of -# writing) but you want to know when infected hosts on your network -# scans hosts on the outside. (Remember that watching for outgoing -# malicious packets is often just as important as watching for incoming -# ones, especially in this case.) The rule currently looks like -# "alert icmp $EXTERNAL_NET any -> $HOME_NET any ..." -# but we want to switch that so it becomes -# "alert icmp $HOME_NET any -> $EXTERNAL_NET any ...". -# Here is how it could be done. -# modifysid 483 \ -# "(.+) \$EXTERNAL_NET (.+) \$HOME_NET (.+)" | \ -# "${1} \$HOME_NET ${2} \$EXTERNAL_NET ${3}" - -# The wildcard (modifysid * ...) can be used to do all kinds of -# interesting things. The substitution expression will be applied on all -# matching rules. First, a silly example to replace "foo" with "bar" in -# all rules (that have the string "foo" in them, that is.) -# modifysid * "foo" | "bar" - -# If you for some reason don't want to use the stream preprocessor to -# match established streams, you may want to replace the 'flow' -# statement with 'flags:A+;' in all those rules. -# modifysid * "flow:[a-z,_ ]+;" | "flags:A+;" - -# Example to convert all rules of classtype attempted-admin to 'drop' -# rules (for Snort_inline only, obviously). -# modifysid * "^alert (.*classtype\s*:\s*attempted-admin)" | "drop ${1}" - -# This one will append some text to the 'msg' string for all rules that -# have the 'tag' keyword in them. -# modifysid * "(.*msg:\s*".+?)"(\s*;.+;\s*tag:.*)" | \ -# "${1}, going to tag this baby"${2}" - -# There may be times when you want to replace multiple occurrences of a -# certain keyword/string in a rule and not just the first one. To -# replace the first two occurrences of "foo" with "bar" in SID 100, -# simply repeat the modifysid statement: -# modifysid 100 "foo" | "bar" -# modifysid 100 "foo" | "bar" - -# Or you can even specify a SID list but repeat the same SID as many -# times as required, like: -# modifysid 100,100,100 "foo" | "bar" - -# Enable all rules in the file exploit.rules. -# modifysid exploit.rules "^#" | "" - -# Enable all rules in exploit.rules, icmp-info.rules and also SID 1171. -# modifysid exploit.rules, snmp.rules, 1171 "^#" | "" - - - -######################################################################## -# SIDs that we don't want to update. # -# If you for some reason don't want a specific rule to be updated # -# (e.g. you made local modifications to it and you never want to # -# update it and don't care about changes in the official version), you # -# can specify a "localsid" statement for it. This means that the old # -# version of the rule (i.e. the one in the rules file on your # -# harddrive) is always kept, regardless if the official version has # -# been updated. Please do not use this feature unless in special # -# cases as it's easy to end up with many signatures that aren't # -# maintained anymore. See the FAQ for details about this and hints # -# about better solutions regarding customization of rules. # -# # -# Syntax: localsid SID # -# or: localsid SID1, SID2, SID3, ... # -######################################################################## - -# Example to never update SID 1325. -# localsid 1325 - - - -######################################################################## -# SIDs to enable after each update. # -# Will simply remove all the leading '#' for a specified SID (if it's # -# a multi-line rule, the leading '#' for all lines are removed.) # -# These will be processed after all the modifysid and disablesid # -# statements. Using 'enablesid' on a rule that is not disabled is a # -# NOOP. # -# # -# Syntax: enablesid SID # -# or: enablesid SID1, SID2, SID3, ... # -######################################################################## - -# Example to enable SID 1325. -# enablesid 1325 - - - -######################################################################## -# SIDs to comment out, i.e. disable, after each update by placing a # -# '#' in front of the rule (if it's a multi-line rule, it will be put # -# in front of all lines). # -# # -# Syntax: disablesid SID # -# or: disablesid SID1, SID2, SID3, ... # -######################################################################## - -# You can specify one SID per line. -# disablesid 1 -# disablesid 2 -# disablesid 3 - -# And also as comma-separated lists. -# disablesid 4,5,6 - -# It's a good idea to also add comment about why you disable the sid: -# disablesid 1324 # 20020101: disabled this SID just because I can diff --git a/config/rootfiles/common/oinkmaster b/config/rootfiles/common/oinkmaster deleted file mode 100644 index 2557353fa..000000000 --- a/config/rootfiles/common/oinkmaster +++ /dev/null @@ -1,2 +0,0 @@ -usr/local/bin/oinkmaster.pl -var/ipfire/suricata/oinkmaster.conf diff --git a/lfs/oinkmaster b/lfs/oinkmaster deleted file mode 100644 index 51b99ecec..000000000 --- a/lfs/oinkmaster +++ /dev/null @@ -1,79 +0,0 @@ -############################################################################### -# # -# IPFire.org - A linux based firewall # -# Copyright (C) 2007-2018 IPFire Team # -# # -# This program is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# This program is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with this program. If not, see . # -# # -############################################################################### - -############################################################################### -# Definitions -############################################################################### - -include Config - -VER = 2.0 -THISAPP = oinkmaster-$(VER) -DL_FILE = $(THISAPP).tar.gz -DL_FROM = $(URL_IPFIRE) -DIR_APP = $(DIR_SRC)/$(THISAPP) -TARGET = $(DIR_INFO)/$(THISAPP) - -############################################################################### -# Top-level Rules -############################################################################### - -objects = $(DL_FILE) - -$(DL_FILE) = $(DL_FROM)/$(DL_FILE) - -$(DL_FILE)_MD5 = fd37d0391ed7b40b84a1b7907cb89508 - -install : $(TARGET) - -check : $(patsubst %,$(DIR_CHK)/%,$(objects)) - -download :$(patsubst %,$(DIR_DL)/%,$(objects)) - -md5 : $(subst %,%_MD5,$(objects)) - -############################################################################### -# Downloading, checking, md5sum -############################################################################### - -$(patsubst %,$(DIR_CHK)/%,$(objects)) : - @$(CHECK) - -$(patsubst %,$(DIR_DL)/%,$(objects)) : - @$(LOAD) - -$(subst %,%_MD5,$(objects)) : - @$(MD5) - -############################################################################### -# Installation Details -############################################################################### - -$(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) - @$(PREBUILD) - @rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar zxf $(DIR_DL)/$(DL_FILE) - cd $(DIR_APP) && patch -Np1 < $(DIR_SRC)/src/patches/oinkmaster-2.0-add_community_rules.patch - cd $(DIR_APP) && chown nobody:nobody oinkmaster.pl - cd $(DIR_APP) && install -m 0644 $(DIR_SRC)/config/oinkmaster/oinkmaster.conf \ - /var/ipfire/suricata/ - cd /var/ipfire/suricata && patch -Np1 < $(DIR_SRC)/src/patches/oinkmaster-tmp.patch - cd $(DIR_APP) && install -m 0755 oinkmaster.pl /usr/local/bin/ - @rm -rf $(DIR_APP) - @$(POSTBUILD) diff --git a/make.sh b/make.sh index 46a6dc8fd..2abd5750c 100755 --- a/make.sh +++ b/make.sh @@ -1450,7 +1450,6 @@ buildipfire() { lfsmake2 ragel lfsmake2 hyperscan lfsmake2 suricata - lfsmake2 oinkmaster lfsmake2 ids-ruleset-sources lfsmake2 squid lfsmake2 squidguard From b645f7fc8675a9caa014b83dff6e7d012a4802c8 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 15:12:58 +0200 Subject: [PATCH 55/77] ids.cgi: Do not longer use hard-coded status messages in oinkmaster_web() function. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 14 +++++++------- langs/de/cgi-bin/de.pl | 5 +++++ langs/en/cgi-bin/en.pl | 7 +++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 053520b57..1d4c3928c 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1862,34 +1862,34 @@ sub oinkmaster_web () { print "

    \n"; # Cleanup the rules directory before filling it with the new rulests. - &_add_to_notice("Remove old rule structures..."); + &_add_to_notice("$Lang::tr{'ids remove rule structures'}"); &IDS::_cleanup_rulesdir(); # Loop through the array of enabled providers. foreach my $provider (@enabled_providers) { - &_add_to_notice("Extracting ruleset for provider: $provider"); + &_add_to_notice("$Lang::tr{'ids extract ruleset'} $provider"); # Call the extractruleset function. &IDS::extractruleset($provider); } # Call function to process the ruleset and do all modifications. - &_add_to_notice("Adjust rules and add user defined customizations..."); + &_add_to_notice("$Lang::tr{'ids adjust ruleset'}"); &IDS::process_ruleset(@enabled_providers); # Call function to merge the classification files. - &_add_to_notice("Merging classifications ..."); + &_add_to_notice("$Lang::tr{'ids merge classifications'}"); &IDS::merge_classifications(@enabled_providers); # Call function to merge the sid to message mapping files. - &_add_to_notice("Merging sid to message files ..."); + &_add_to_notice("$Lang::tr{'ids merge sid files'}"); &IDS::merge_sid_msg(@enabled_providers); # Cleanup temporary directory. - &_add_to_notice("Cleanup temporary directory ..."); + &_add_to_notice("$Lang::tr{'ids cleanup tmp dir'}"); &IDS::cleanup_tmp_directory(); # Finished - print a notice. - &_add_to_notice("Finished..."); + &_add_to_notice("$Lang::tr{'ids finished'}"); # Close the working notice. &_close_working_notice(); diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 0be097609..148c4323b 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1377,15 +1377,19 @@ 'idle timeout' => 'Leerlaufwartezeit in Minuten (0 zum Deaktivieren):', 'idle timeout not set' => 'Leerlaufwartezeit nicht angegeben.', 'ids add provider' => 'Provider hinzufügen', +'ids adjust ruleset' => 'Regelset anpassen und Benutzermodifikationen übernehmen...', 'ids apply' => 'Übernehmen', 'ids apply ruleset changes' => 'Regeländerungen werden übernommen. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids automatic rules update' => 'Automatische Regelaktualisierung', +'ids cleanup tmp dir' => 'Temporäres Verzeichnis aufräumen...', 'ids autoupdates' => 'Automatische Updates', 'ids could not add provider' => 'Provider konnte nicht hinzugefügt werden', 'ids customize ruleset' => 'Regelset anpassen', 'ids download new ruleset' => 'Das neue Regelset wird heruntergeladen und entpackt. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids enable' => 'Einbruchsverhinderungssystem aktivieren', 'ids enable automatic updates' => 'Automatische Updates aktivieren', +'ids extract ruleset' => 'Entpacke Regelset von:', +'ids finished' => 'Fertig...', 'ids force ruleset update' => 'Regelset jetzt aktualisieren', 'ids hide' => 'Verstecken', 'ids ignored hosts' => 'Ausnahmeliste', @@ -1399,6 +1403,7 @@ 'ids oinkcode required' => 'Für den ausgewählten Regelsatz wird ein Abonnement oder ein Oinkcode benötigt', 'ids provider' => 'Regelset-Anbieter', 'ids provider settings' => 'Regelset-Anbieter-Einstellungen', +'ids remove rule structures' => 'Entferne alte Regelstrukturen...', 'ids reset provider' => 'Providereinstellungen zurücksetzen', 'ids rules update' => 'Regelsatz', 'ids ruleset autoupdate in progress' => 'Der Regelsatz wird gerade aktualisiert. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index d98422621..975b3f58d 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1421,27 +1421,34 @@ 'idle timeout' => 'Idle timeout (mins; 0 to disable):', 'idle timeout not set' => 'Idle timeout not set.', 'ids add provider' => 'Add provider', +'ids adjust ruleset' => 'Adjust rules and add user defined customizations...', 'ids apply' => 'Apply', 'ids apply ruleset changes' => 'The ruleset changes are being applied. Please wait until all operations have completed successfully...', 'ids automatic rules update' => 'Automatic Rule Update', 'ids autoupdates' => 'Automatic updates', +'ids cleanup tmp dir' => 'Cleanup temporary directory...', 'ids could not add provider' => 'Could not add provider', 'ids customize ruleset' => 'Customize ruleset', 'ids download new ruleset' => 'Downloading and unpacking new ruleset. Please wait until all operations have completed successfully...', 'ids enable' => 'Enable Intrusion Prevention System', 'ids enable automatic updates' => 'Enable automatic updates', +'ids extract ruleset' => 'Extracting ruleset for provider:', +'ids finished' => 'Finished...', 'ids force ruleset update' => 'Force ruleset update', 'ids hide' => 'Hide', 'ids ignored hosts' => 'Whitelisted Hosts', 'ids log hits' => 'Total of number of activated rules for', 'ids log viewer' => 'IPS Log Viewer', 'ids logs' => 'IPS Logs', +'ids merge classifications' => 'Merging classifications...', +'ids merge sid files' => 'Merging sid to message files...', 'ids monitor traffic only' => 'Monitor traffic only', 'ids monitored interfaces' => 'Monitored Interfaces', 'ids no enabled ruleset provider' => 'No enabled ruleset is available. Please activate or add one first.', 'ids no network zone' => 'Please select at least one network zone to be monitored', 'ids provider' => 'Provider', 'ids provider settings' => 'Provider settings', +'ids remove rule structures' => 'Remove old rule structures...', 'ids reset provider' => 'Reset provider', 'ids rules update' => 'Ruleset', 'ids ruleset autoupdate in progress' => 'Ruleset update in progress. Please wait until all operations have completed successfully...', From 00271ed769a64e309498c8c5ab2267c0e5982957 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 15:30:03 +0200 Subject: [PATCH 56/77] ids.cgi: Handle "Not modified" when forcing an ruleset update. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 12 ++++++++++-- langs/de/cgi-bin/de.pl | 1 + langs/en/cgi-bin/en.pl | 1 + 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 1d4c3928c..a16d4fa0d 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -436,8 +436,16 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &working_notice("$Lang::tr{'ids download new ruleset'}"); # Call subfunction to download the ruleset. - if(&IDS::downloadruleset($provider)) { - $errormessage = "$provider - $Lang::tr{'could not download latest updates'}"; + my $return = &IDS::downloadruleset($provider); + + # Check if the download function gives a return code. + if ($return) { + # Handle different return codes. + if ($return eq "not modified") { + $errormessage = "$provider - $Lang::tr{'ids ruleset is up to date'}"; + } else { + $errormessage = "$provider - $Lang::tr{'could not download latest updates'}: $return"; + } # Call function to store the errormessage. &IDS::_store_error_message($errormessage); diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 148c4323b..62a3ee380 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1407,6 +1407,7 @@ 'ids reset provider' => 'Providereinstellungen zurücksetzen', 'ids rules update' => 'Regelsatz', 'ids ruleset autoupdate in progress' => 'Der Regelsatz wird gerade aktualisiert. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', +'ids ruleset is up to date' => 'Regelset ist aktuell - Keine Aktualisierung notwendig.', 'ids ruleset settings' => 'Regelsatzeinstellungen', 'ids show' => 'Anzeigen', 'ids the choosen provider is already in use' => 'Der gewhählte Provider wird bereits verwendet.', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index 975b3f58d..0c6ebc881 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1452,6 +1452,7 @@ 'ids reset provider' => 'Reset provider', 'ids rules update' => 'Ruleset', 'ids ruleset autoupdate in progress' => 'Ruleset update in progress. Please wait until all operations have completed successfully...', +'ids ruleset is up to date' => 'No update required - The ruleset is up to date.', 'ids ruleset settings' => 'Ruleset Settings', 'ids show' => 'Show', 'ids subscription code required' => 'The selected ruleset requires a subscription code', From 5bad33e9a4bae9e15979087df3420c30dd5afd6c Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 15:32:27 +0200 Subject: [PATCH 57/77] ids.cgi: Display return code on download error, when adding a new provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index a16d4fa0d..3a408ff5d 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -755,8 +755,11 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &working_notice("$Lang::tr{'ids working'}"); # Download the ruleset. - if(&IDS::downloadruleset($provider)) { - $errormessage = "$Lang::tr{'ids could not add provider'} - $Lang::tr{'ids unable to download the ruleset'}"; + my $return = &IDS::downloadruleset($provider); + + # Check if the downloader returned a code. + if ($return) { + $errormessage = "$Lang::tr{'ids could not add provider'} - $Lang::tr{'ids unable to download the ruleset'}: $return"; # Call function to store the errormessage. &IDS::_store_error_message($errormessage); From f7eedacb43e81dd8acd031f1ed7680fd0bf3b2b9 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 15:51:06 +0200 Subject: [PATCH 58/77] convert-ids-backend-files: Restart suricata if the IDS is running. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 302ed5232..825a32c79 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -243,3 +243,13 @@ foreach my $file (@files_to_drop) { # Remove the file if it exists. unlink("$file") if (-f "$file"); } + +# +## Step 8: Restart the IDS if running. +# + +# Check if the IDS is running. +if(&IDS::ids_is_running()) { + # Call suricatactrl to perform the restart. + &IDS::call_suricatactrl("restart"); +} From 7bc15b982c7ce3bd0b6d3cf752e1e42abba4fe1d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 15:54:44 +0200 Subject: [PATCH 59/77] backup: Add files for new IDS backend. Signed-off-by: Stefan Schantl --- config/backup/include | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/config/backup/include b/config/backup/include index 809a49601..354998eea 100644 --- a/config/backup/include +++ b/config/backup/include @@ -23,6 +23,7 @@ etc/unbound root/.bash_history root/.gitconfig root/.ssh +var/cache/suricata var/ipfire/auth/users var/ipfire/backup/addons/backup var/ipfire/backup/exclude.user @@ -47,9 +48,7 @@ var/ipfire/ppp var/ipfire/proxy var/ipfire/qos/* var/ipfire/qos/bin/qos.sh -var/ipfire/suricata/*.conf -var/ipfire/suricata/*.yaml -var/ipfire/suricata/providers-settings +var/ipfire/suricata var/ipfire/*/settings var/ipfire/time/ var/ipfire/urlfilter @@ -59,5 +58,3 @@ var/log/ip-acct/* var/log/rrd/* var/log/rrd/collectd var/log/vnstat -var/tmp/idsrules-*.tar.gz -var/tmp/idsrules-*.rules From b3dbe9ef6462b90198f969dcf42bb17f9c4b427f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 15:57:34 +0200 Subject: [PATCH 60/77] backup.pl: Run convert-ids-backend-files converter. Signed-off-by: Stefan Schantl --- config/backup/backup.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/backup/backup.pl b/config/backup/backup.pl index a2337cf23..79bf94cc9 100644 --- a/config/backup/backup.pl +++ b/config/backup/backup.pl @@ -171,6 +171,12 @@ restore_backup() { convert-ids-multiple-providers fi + # IDS backend converter. + if [ -e "/var/ipfire/suricata/oinkmaster.conf" ]; then + # Run the converter + convert-ids-backend-files + fi + # Convert DNS settings convert-dns-settings From da5c7c24f022751ff4d8dfb68c65d0e60801a626 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 16 Apr 2022 16:02:28 +0200 Subject: [PATCH 61/77] ids.cgi: Remove orphaned headline. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 3a408ff5d..cdfc77ac0 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1105,17 +1105,6 @@ END print < - - -

    -

    -

    -

    - - - - $Lang::tr{'ids automatic rules update'} -

    From 7c4b8df7163e60bc05867531e3d2a7001eb2af59 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 17 Apr 2022 15:02:41 +0200 Subject: [PATCH 62/77] update-ids-ruleset: Skip unsupported providers. In case a configured provider is not longer supported, simply skip it and do not try to perform an update. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index c62bdcb3b..e9a082e62 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -26,6 +26,9 @@ require '/var/ipfire/general-functions.pl'; require "${General::swroot}/ids-functions.pl"; require "${General::swroot}/lang.pl"; +# Import ruleset providers file. +require "$IDS::rulesetsourcesfile"; + # Load perl module to talk to the kernel syslog. use Sys::Syslog qw(:DEFAULT setlogsock); @@ -102,6 +105,9 @@ foreach my $id (keys %providers) { my $enabled_status = $providers{$id}[2]; my $autoupdate_status = $providers{$id}[3]; + # Skip unsupported providers. + next unless($IDS::Ruleset::Providers{$provider}); + # Skip the provider if it is not enabled. next unless($enabled_status eq "enabled"); From 6bef05b9ed1eacb57f66f565def49bbfe6400946 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 17 Apr 2022 15:03:56 +0200 Subject: [PATCH 63/77] ids.cgi: Proper handle providers which are not longer supported. They will be shown with a different background colour to get the users attention. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index cdfc77ac0..2a837bfc1 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1161,6 +1161,16 @@ END $col="bgcolor='$color{'color20'}'"; } + # Handle providers which are not longer supported. + unless ($provider_name) { + # Set the provider name to the provider handle + # to display something helpful. + $provider_name = $provider; + + # Assign background color + $col="bgcolor='#FF4D4D'"; + } + # Choose icons for the checkboxes. my $status_gif; my $status_gdesc; @@ -2031,6 +2041,9 @@ sub get_provider_name($) { my ($handle) = @_; my $provider_name; + # Early exit if the given provider does not longer exist. + return unless ($IDS::Ruleset::Providers{$handle}); + # Get the required translation string for the given provider handle. my $tr_string = $IDS::Ruleset::Providers{$handle}{'tr_string'}; From eaf5364413ab44dff0640396653fef4e39ace4d7 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 17 Apr 2022 15:21:20 +0200 Subject: [PATCH 64/77] ids.cgi: Disable manual update button if a provider is not longer supported. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 2a837bfc1..3ee66e06b 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1780,7 +1780,8 @@ END ## Function to show the area where additional provider actions can be done. # sub show_additional_provider_actions() { - my $disabled; + my $disabled_reset; + my $disabled_update; my %used_providers = (); # Read-in providers settings file. @@ -1795,7 +1796,12 @@ sub show_additional_provider_actions() { # Disable the reset provider button if no provider modified sids file exists. unless (-f $modifications_file) { - $disabled = "disabled"; + $disabled_reset = "disabled"; + } + + # Disable the manual update button if the provider is not longer supported. + unless ($IDS::Ruleset::Providers{$provider}) { + $disabled_update = "disabled"; } &Header::openbox('100%', 'center', ""); @@ -1805,8 +1811,8 @@ sub show_additional_provider_actions() { - - + + From 38cf581405290ac9781793e8785cbdf0e210dced Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 17 Apr 2022 16:38:21 +0200 Subject: [PATCH 65/77] ids-functions.pl: Remove temporary files if the downloader aborts. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index c916926de..41a07897b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -427,6 +427,9 @@ sub downloadruleset ($) { # Check if the server responds with 304 (Not Modified). } elsif ($response->code == 304) { + # Remove temporary file, if one exists. + unlink("$tmpfile") if (-e "$tmpfile"); + # Return "not modified". return "not modified"; @@ -435,6 +438,9 @@ sub downloadruleset ($) { # Obtain error. my $error = $response->content; + # Remove temporary file, if one exists. + unlink("$tmpfile") if (-e "$tmpfile"); + # Return the error message from response.. return "$error"; } From 782418e226434fbd7fbd236699a45bce328dcd6d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 19 Apr 2022 15:10:31 +0200 Subject: [PATCH 66/77] Add missing german translation strings. Signed-off-by: Stefan Schantl --- langs/de/cgi-bin/de.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 62a3ee380..abc11752d 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1396,6 +1396,8 @@ 'ids log hits' => 'Gesamtanzahl der Regeltreffer für', 'ids log viewer' => 'Protokoll des Einbruchsverhinderungssystems', 'ids logs' => 'IPS-Protokolldateien', +'ids merge classifications' => 'Klassifizierungen zusammenführen...', +'ids merge sid files' => 'Sid-to-message Dateien zusammenführen...', 'ids monitor traffic only' => 'Netzwerkpakete nur überprüfen (nicht verwerfen)', 'ids monitored interfaces' => 'Überwachte Netzwerkzonen', 'ids no enabled ruleset provider' => 'Es ist kein aktivierter Provider verfügbar. Bitte aktivieren Sie einen oder fügen Sie einen Provider hinzu.', From c62121c7e4ef9ec5688e16b04ef59e21276e1bd0 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 20 Apr 2022 20:58:04 +0200 Subject: [PATCH 67/77] ids-functions.pl: Try to enumerate the dl_rulesfile if a provider is not supported anymore. In this case the details about the file suffix is not available in the ruleset-sources file anymore. In this case now the function tries to enumerate the correct filename. This allows to display the correct stats in the WUI and to extract and use the downloaded ruleset of the provider until it got deleted by the user. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 50 ++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 41a07897b..f6627ba2b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1027,23 +1027,47 @@ sub _store_error_message ($) { sub _get_dl_rulesfile($) { my ($provider) = @_; - # Gather the download type for the given provider. - my $dl_type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; + # Check if the requested provider is known. + if ($IDS::Ruleset::Providers{$provider}) { + # Gather the download type for the given provider. + my $dl_type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; - # Obtain the file suffix for the download file type. - my $suffix = $dl_type_to_suffix{$dl_type}; + # Obtain the file suffix for the download file type. + my $suffix = $dl_type_to_suffix{$dl_type}; - # Check if a suffix has been found. - unless ($suffix) { - # Abort return - nothing. - return; + # Check if a suffix has been found. + unless ($suffix) { + # Abort return - nothing. + return; + } + + # Generate the full filename and path for the stored rules file. + my $rulesfile = "$dl_rules_path/$dl_rulesfile_prefix-$provider$suffix"; + + # Return the generated filename. + return $rulesfile; + + } else { + # A downloaded ruleset for a provider which is not supported anymore is requested. + # + # Try to enumerate the downloaded ruleset file. + foreach my $dl_type (keys %dl_type_to_suffix) { + # Get the file suffix for the supported type. + my $suffix = $dl_type_to_suffix{$dl_type}; + + # Generate possible ruleset file name. + my $rulesfile = "$dl_rules_path/$dl_rulesfile_prefix-$provider$suffix"; + + # Check if such a file exists. + if (-f $rulesfile) { + # Downloaded rulesfile found - Return the filename. + return $rulesfile; + } + } } - # Generate the full filename and path for the stored rules file. - my $rulesfile = "$dl_rules_path/$dl_rulesfile_prefix-$provider$suffix"; - - # Return the generated filename. - return $rulesfile; + # If we got here, no rulesfile could be determined - return nothing. + return; } # From e41ee3e0f24cb89b20a758e2281531ed76577ef4 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 22 Apr 2022 05:31:28 +0200 Subject: [PATCH 68/77] ids-functions.pl: Avoid suricata from loading rulesfiles of an unsupported provider. Modify the write_used_rulefiles_file() function to skip the rulesfiles of unsupported providers. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index f6627ba2b..3994d1780 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1484,6 +1484,9 @@ sub write_used_rulefiles_file (@) { # Loop through the array of enabled providers. foreach my $provider (@providers) { + # Skip unsupported providers. + next unless ($IDS::Ruleset::Providers{$provider}); + # Get the used rulefile for this provider. my @used_rulesfiles = &get_provider_used_rulesfiles($provider); From 07dc722f611685c6018630f927ad4b65f44988d1 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 22 Apr 2022 05:44:23 +0200 Subject: [PATCH 69/77] ids.cgi: Make the page lock in oinkmaster_web() function optional. This allows to call and release the page lock manually. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 3ee66e06b..c961c91bf 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1867,10 +1867,14 @@ sub _close_working_notice () { ## oinkmaster function, but provides a lot of HTML formated status output. # sub oinkmaster_web () { + my ($nolock) = @_; + my @enabled_providers = &IDS::get_enabled_providers(); # Lock the webpage and print message. - &_open_working_notice("$Lang::tr{'ids apply ruleset changes'}"); + unless ($nolock) { + &_open_working_notice("$Lang::tr{'ids apply ruleset changes'}"); + } # Check if the files in rulesdir have the correct permissions. &IDS::_check_rulesdir_permissions(); @@ -1908,7 +1912,9 @@ sub oinkmaster_web () { &_add_to_notice("$Lang::tr{'ids finished'}"); # Close the working notice. - &_close_working_notice(); + unless ($nolock) { + &_close_working_notice(); + } } # From 1febad2ad41578d3e77195929076e3cbbc28a89f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 22 Apr 2022 05:45:56 +0200 Subject: [PATCH 70/77] ids.cgi: Avoid doubble locking the page when forcing a ruleset update. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index c961c91bf..3a0f1fc97 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -433,7 +433,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { unless ($errormessage) { # Lock the webpage and print notice about downloading # a new ruleset. - &working_notice("$Lang::tr{'ids download new ruleset'}"); + &_open_working_notice("$Lang::tr{'ids download new ruleset'}"); # Call subfunction to download the ruleset. my $return = &IDS::downloadruleset($provider); @@ -450,11 +450,17 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Call function to store the errormessage. &IDS::_store_error_message($errormessage); + # Close the working notice. + &_close_working_notice(); + # Preform a reload of the page. &reload(); } else { # Call subfunction to launch oinkmaster. - &oinkmaster_web(); + &oinkmaster_web("nolock"); + + # Close the working notice. + &_close_working_notice(); # Check if the IDS is running. if(&IDS::ids_is_running()) { From c8adaee1958ed0c382341e08949d5cb88bd58c7e Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 22 Apr 2022 05:47:21 +0200 Subject: [PATCH 71/77] ruleset-sources: Remove support for PT Attack Team Detection rules. All of a sudden this ruleset provider has dissapeared from Github. I was not able to find any further details or web page or the ruleset anymore. Signed-off-by: Stefan Schantl --- config/suricata/ruleset-sources | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index 1d2c6e98b..54a739d18 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -97,16 +97,6 @@ our %Providers = ( dl_type => "plain", }, - # Positive Technologies Attack Detection Team rules. - attack_detection => { - summary => "PT Attack Detection Team Rules", - website => "https://github.com/ptresearch/AttackDetection", - tr_string => "attack detection team rules", - requires_subscription => "False", - dl_url => "https://raw.githubusercontent.com/ptresearch/AttackDetection/master/pt.rules.tar.gz", - dl_type => "archive", - }, - # Secureworks Security rules. secureworks_security => { summary => "Secureworks Security Ruleset", From 615fd78f9294b2843e396f3e70b2181d8491725d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 22 Apr 2022 09:13:41 +0200 Subject: [PATCH 72/77] convert-ids-backend-files: Set correct ownership for suricata used rulefiles file. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 825a32c79..982808d47 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -234,6 +234,9 @@ my @enabled_providers = &IDS::get_enabled_providers(); # Write used rulesfiles file. &IDS::write_used_rulefiles_file(@enabled_providers); +# Set the correct ownership for the new file. +&IDS::set_ownership("$IDS::suricata_used_rulesfiles_file"); + # ## Step 7: Remove unneeded orphaned files. # From adce5b1c8fc21916c77d7e8a40cbed2baac1f2a2 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 22 Apr 2022 13:31:51 +0200 Subject: [PATCH 73/77] convert-ids-backend-files: Stop and start suricata during runtime. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 37 ++++++++++++++++------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 982808d47..55c3afbf8 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -46,7 +46,22 @@ my @files_to_drop = ( ); # -# Step 1: Move downloaded files to new location. +## Step 1: Stop suricata if it is running. +# +my $start_suricata; + +# Check if the IDS is running. +if(&IDS::ids_is_running()) { + # Call suricatactrl to stop the IDS. + &IDS::call_suricatactrl("stop"); + + # Set start_suricata to true to start it + # at the end of the script again. + $start_suricata = "1"; +} + +# +## Step 2: Move downloaded files to new location. # my $old_dl_rulesfiles_dir = "/var/tmp"; @@ -79,7 +94,7 @@ closedir(DIR); my @providers = &IDS::get_ruleset_providers(); # -## Step 2: Convert used rules files. +## Step 3: Convert used rules files. # # Loop through the array of known providers. @@ -128,7 +143,7 @@ foreach my $provider (@providers) { } # -## Step 3: Convert ruleset modifictaion files. +## Step 4: Convert ruleset modifictaion files. # # Loop through the array of providers. @@ -184,7 +199,7 @@ foreach my $provider (@providers) { } # -## Step 4: Convert MONTIOR_TRAFFIC_ONLY setting. +## Step 5: Convert MONTIOR_TRAFFIC_ONLY setting. # my %ids_settings = (); @@ -217,7 +232,7 @@ foreach my $key (keys %provider_settings) { &General::writehasharray("$IDS::providers_settings_file", \%provider_settings); # -## Step 5: Regenerate the ruleset. +## Step 6: Regenerate the ruleset. # # @@ -225,7 +240,7 @@ foreach my $key (keys %provider_settings) { &IDS::oinkmaster(); # -## Step 6: Write new config file for suricata which contains the used rulesfiles. +## Step 7: Write new config file for suricata which contains the used rulesfiles. # # Get enabled providers. @@ -238,7 +253,7 @@ my @enabled_providers = &IDS::get_enabled_providers(); &IDS::set_ownership("$IDS::suricata_used_rulesfiles_file"); # -## Step 7: Remove unneeded orphaned files. +## Step 8: Remove unneeded orphaned files. # # Loop through the array of files which are safe to drop. @@ -248,11 +263,11 @@ foreach my $file (@files_to_drop) { } # -## Step 8: Restart the IDS if running. +## Step 9: Start the IDS again, if it was running. # # Check if the IDS is running. -if(&IDS::ids_is_running()) { - # Call suricatactrl to perform the restart. - &IDS::call_suricatactrl("restart"); +if($start_suricata) { + # Call suricatactrl to perform the start of the IDS. + &IDS::call_suricatactrl("start"); } From 249d796b4b873fde6e4bf270b7028afe8073abc2 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 25 Apr 2022 20:12:19 +0200 Subject: [PATCH 74/77] convert-ids-backend-files: Wait until suricata has stopped sucessfully. Signed-off-by: Stefan Schantl --- config/suricata/convert-ids-backend-files | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/suricata/convert-ids-backend-files b/config/suricata/convert-ids-backend-files index 55c3afbf8..614307380 100644 --- a/config/suricata/convert-ids-backend-files +++ b/config/suricata/convert-ids-backend-files @@ -58,6 +58,9 @@ if(&IDS::ids_is_running()) { # Set start_suricata to true to start it # at the end of the script again. $start_suricata = "1"; + + # Wait until suricata has stopped. + sleep 1 while (-f $IDS::idspidfile); } # From 91a8664b662ed506a7896b638c6d9d140485a5aa Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 25 Apr 2022 21:15:23 +0200 Subject: [PATCH 75/77] Revert "ruleset-sources: Remove support for PT Attack Team Detection rules." The ruleset provider has recovered his github presence. This reverts commit c8adaee1958ed0c382341e08949d5cb88bd58c7e. --- config/suricata/ruleset-sources | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index 54a739d18..1d2c6e98b 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -97,6 +97,16 @@ our %Providers = ( dl_type => "plain", }, + # Positive Technologies Attack Detection Team rules. + attack_detection => { + summary => "PT Attack Detection Team Rules", + website => "https://github.com/ptresearch/AttackDetection", + tr_string => "attack detection team rules", + requires_subscription => "False", + dl_url => "https://raw.githubusercontent.com/ptresearch/AttackDetection/master/pt.rules.tar.gz", + dl_type => "archive", + }, + # Secureworks Security rules. secureworks_security => { summary => "Secureworks Security Ruleset", From 504fb53bcc1eb03af782d800b77ee6a1b6e4077b Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 26 Apr 2022 05:23:44 +0200 Subject: [PATCH 76/77] ids-functions.pl: Add remove_from_etags() function. This function is used to drop the stored etags data of a given provider. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 3994d1780..ec4fdacc5 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1884,6 +1884,30 @@ sub get_provider_used_rulesfiles($) { return @used_rulesfiles; } +# +## Function to delete the stored etag data of a given provider. +# +sub remove_from_etags ($) { + my ($provider) = @_; + + my %etags; + + # Early exit function if the etags file does not exist. + return unless (-f $etags_file); + + # Read-in etag file. + &General::readhash("$etags_file", \%etags); + + # Check if the hash contains an entry for the given provider. + if ($etags{$provider}) { + # Drop the entry. + delete($etags{$provider}); + + # Write back the etags file. + &General::writehash("$etags_file", \%etags); + } +} + # ## Function to write the lock file for locking the WUI, while ## the autoupdate script runs. From 1a9e81ce7f999628536c5fa33928f3e79a7d84cc Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 26 Apr 2022 05:24:47 +0200 Subject: [PATCH 77/77] ids.cgi: Remove etag data when deleting a provider. Otherwise the same provider could not be added again at a later time if the stored etag is still valid. In this case the server will not offer the rules and the provider could not be added. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 3a0f1fc97..369bf0276 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -906,6 +906,9 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Drop the stored ruleset file. &IDS::drop_dl_rulesfile($provider); + # Remove may stored etag data. + &IDS::remove_from_etags($provider); + # Get the name of the provider rulessets include file. my $provider_used_rulefile = &IDS::get_provider_used_rulesfiles_file($provider);