From cf3806f27ca0d53ed0e1c28e4e23e4cd53816da6 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 12 May 2021 20:05:55 +0200 Subject: [PATCH 001/140] ddns: Add upstream patch to fix argparse list-token-providers command. Fixes #12607. Signed-off-by: Stefan Schantl --- lfs/ddns | 1 + ...13-fix-argparse-list-token-providers.patch | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 src/patches/ddns-013-fix-argparse-list-token-providers.patch diff --git a/lfs/ddns b/lfs/ddns index 98b374ed1..538cf7256 100644 --- a/lfs/ddns +++ b/lfs/ddns @@ -77,6 +77,7 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects)) cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/ddns-013-duckdns-new-api.patch cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/ddns-013-add-option-to-list-token-provider.patch cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/ddns-013-proper-encode-string.patch + cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/ddns-013-fix-argparse-list-token-providers.patch cd $(DIR_APP) && [ -x "configure" ] || sh ./autogen.sh cd $(DIR_APP) && ./configure \ diff --git a/src/patches/ddns-013-fix-argparse-list-token-providers.patch b/src/patches/ddns-013-fix-argparse-list-token-providers.patch new file mode 100644 index 000000000..065dbb666 --- /dev/null +++ b/src/patches/ddns-013-fix-argparse-list-token-providers.patch @@ -0,0 +1,23 @@ +commit 5e075681008174839a47cae698ae459c0ea3a30a +Author: Stefan Schantl +Date: Wed May 12 19:59:01 2021 +0200 + + Fix argsparse string for listing token providers. + + Signed-off-by: Stefan Schantl + +diff --git a/ddns.in b/ddns.in +index 538e4b0..20edd28 100755 +--- a/ddns.in ++++ b/ddns.in +@@ -49,8 +49,8 @@ def main(): + p_list_providers = subparsers.add_parser("list-providers", + help=_("List all available providers")) + +- # list-token-provider +- p_list_token_provider = subparsers.add_parser("list-token-provider", ++ # list-token-providers ++ p_list_token_provider = subparsers.add_parser("list-token-providers", + help=_("List all providers which supports authentication via token")) + + # update From 3ed6be83d21ba2c0a819974dfc1936ca7bfff541 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 22 Jun 2021 14:51:35 +0200 Subject: [PATCH 002/140] ovpnmain.cgi: Call correct system_output() function. Signed-off-by: Stefan Schantl --- html/cgi-bin/ovpnmain.cgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/cgi-bin/ovpnmain.cgi b/html/cgi-bin/ovpnmain.cgi index d9e26de2f..e6b272f03 100644 --- a/html/cgi-bin/ovpnmain.cgi +++ b/html/cgi-bin/ovpnmain.cgi @@ -5764,7 +5764,7 @@ END # Adding DH parameter to chart if (-f "${General::swroot}/ovpn/ca/dh1024.pem") { - my @dhsubject = &System_output("/usr/bin/openssl", "dhparam", "-text", "-in", "${General::swroot}/ovpn/ca/dh1024.pem"); + my @dhsubject = &General::system_output("/usr/bin/openssl", "dhparam", "-text", "-in", "${General::swroot}/ovpn/ca/dh1024.pem"); my $dhsubject; foreach my $line (@dhsubject) { From c7ee7f60dac6be66c0b45261fd3aa3c3c6852f69 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 22 Jun 2021 14:56:40 +0200 Subject: [PATCH 003/140] ovpnmain.cgi: Fix typos. Signed-off-by: Stefan Schantl --- html/cgi-bin/ovpnmain.cgi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/html/cgi-bin/ovpnmain.cgi b/html/cgi-bin/ovpnmain.cgi index e6b272f03..7a2833ce6 100644 --- a/html/cgi-bin/ovpnmain.cgi +++ b/html/cgi-bin/ovpnmain.cgi @@ -1235,7 +1235,7 @@ if ($cgiparams{'ACTION'} eq $Lang::tr{'save'} && $cgiparams{'TYPE'} eq '' && $cg if ( $vpnsettings{'ENABLED_BLUE'} eq 'on' ) { &General::system("touch", "${General::swroot}/ovpn/enable_blue"); } else { - unlink(${General::swroot}/ovpn/enable_blue); + unlink("${General::swroot}/ovpn/enable_blue"); } if ( $vpnsettings{'ENABLED_ORANGE'} eq 'on' ) { @@ -1497,7 +1497,7 @@ END } } - @casubject = &General::system_output("/usr/bin/openssl", "x509", "-text", "-in", "${General::swroot}/ovpn/ca/$cgiparams{'CA_NAME'}cert.pem"); + my @casubject = &General::system_output("/usr/bin/openssl", "x509", "-text", "-in", "${General::swroot}/ovpn/ca/$cgiparams{'CA_NAME'}cert.pem"); my $casubject; foreach my $line (@casubject) { @@ -2526,7 +2526,7 @@ else if ($confighash{$cgiparams{'KEY'}}) { # Revoke certificate if certificate was deleted and rewrite the CRL - &General::system("/usr/bin/openssl", "ca", "-revoke", "${General::swroot}/ovpn/certs/$confighash{$cgiparams{'KEY'}}[1]cert.pem", "-config", "${General::swroot}/ovpn/openssl/ovpn.cnf)"; + &General::system("/usr/bin/openssl", "ca", "-revoke", "${General::swroot}/ovpn/certs/$confighash{$cgiparams{'KEY'}}[1]cert.pem", "-config", "${General::swroot}/ovpn/openssl/ovpn.cnf"); &General::system("/usr/bin/openssl", "ca", "-gencrl", "-out", "${General::swroot}/ovpn/crls/cacrl.pem", "-config", "${General::swroot}/ovpn/openssl/ovpn.cnf"); ### From 1c959b88c7ab8b79df61f3e51c633c0455cfb421 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 30 Jul 2021 19:54:15 +0200 Subject: [PATCH 004/140] qos.cgi: Fix truncated status output In the past only the fist line of the status output has been passed to the cleanhtml() function and displayed. Now the whole output will be converted to a string, cleaned and displyed on the WUI again. Fixes #12666. Signed-off-by: Stefan Schantl --- html/cgi-bin/qos.cgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/cgi-bin/qos.cgi b/html/cgi-bin/qos.cgi index fa566b523..d768f95f5 100644 --- a/html/cgi-bin/qos.cgi +++ b/html/cgi-bin/qos.cgi @@ -543,7 +543,7 @@ elsif ($qossettings{'ACTION'} eq $Lang::tr{'status'} ) if ($qossettings{'ENABLED'} eq 'on'){ my $output = ""; my @output = &General::system_output("/usr/local/bin/qosctrl", "status"); - $output = &Header::cleanhtml(@output[0],"y"); + $output = &Header::cleanhtml(join("", @output), "y"); print "
$output
\n"; } else { print "$Lang::tr{'QoS not enabled'}"; } &Header::closebox(); From d0885624067d40da7f6ff26c6be66fc39ab73d12 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 16 Dec 2021 20:04:41 +0100 Subject: [PATCH 005/140] suricata: Do not load rules for dnp3 and modbus. The parsers for those are disabled in the suricata config so the rules are not needed, on the contrary they massively will spam warnings when launching suricate because of the disabled parsers. Signed-off-by: Stefan Schantl --- config/suricata/suricata-default-rules.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/suricata/suricata-default-rules.yaml b/config/suricata/suricata-default-rules.yaml index 64493e462..d6c358add 100644 --- a/config/suricata/suricata-default-rules.yaml +++ b/config/suricata/suricata-default-rules.yaml @@ -5,13 +5,11 @@ - /usr/share/suricata/rules/app-layer-events.rules - /usr/share/suricata/rules/decoder-events.rules - /usr/share/suricata/rules/dhcp-events.rules - - /usr/share/suricata/rules/dnp3-events.rules - /usr/share/suricata/rules/dns-events.rules - /usr/share/suricata/rules/files.rules - /usr/share/suricata/rules/http-events.rules - /usr/share/suricata/rules/ipsec-events.rules - /usr/share/suricata/rules/kerberos-events.rules - - /usr/share/suricata/rules/modbus-events.rules - /usr/share/suricata/rules/nfs-events.rules - /usr/share/suricata/rules/ntp-events.rules - /usr/share/suricata/rules/smb-events.rules From 5e891296f061d5bcccd07147bf1c159703085054 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 19 Mar 2021 21:24:36 +0100 Subject: [PATCH 006/140] ruleset-sources: Rework file format and data. The file now contains a lot more of data and easily can be extended to provide more and new providers. Signed-off-by: Stefan Schantl --- config/suricata/ruleset-sources | 74 ++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index a00cef945..638d32fb8 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -1,15 +1,69 @@ -# Ruleset for registered sourcefire users. -registered = https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode= +package IDS::Ruleset; -# Ruleset for registered sourcefire users with valid subscription. -subscripted = https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode= +# This file contains the supported ruleset providers. +# +# Each one is defined as a hash in the main hash. +# It's name acts as handle/key and the key/value pair acts as data part. +# So the structure is like the following: +# +# handle => { +# summary => A short summary of the service. This also will be shown if no translation string is available for the WUI. +# website => The website of the ruleset provider. +# tr_string => The translation string which is used by the WUI and part of the language files. +# requires_subscription => "True/False" - If some kind of registration code is required in order to download the ruleset. +# dl_url => The download URL to grab the ruleset. +# dl_type => "archive/file" - To specify, if the downloaded file is a packed archive or a plain text file. +# }, -# Community rules from sourcefire. -community = https://www.snort.org/rules/community +# Hash which contains the supported ruleset providers. +our %Providers = ( + # Ruleset for registered sourcefire users. + registered => { + summary => "Talos VRT rules for registered users", + website => "https://www.snort.org", + tr_string => "registered user rules", + requires_subscription => "True", + dl_url => "https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode=", + dl_type => "archive", + }, -# Emerging threads community rules. -emerging = https://rules.emergingthreats.net/open/suricata-5.0/emerging.rules.tar.gz + # Ruleset for registered sourcefire users with a valid subsription. + subscripted => { + summary => "Talos VRT rules with subscription", + website => "https://www.snort.org", + tr_string => "subscripted user rules", + requires_subscription => "True", + dl_url => "https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode=", + dl_type => "archive", + }, -# Emerging threads pro rules. -emerging_pro = https://rules.emergingthreatspro.com//suricata-5.0/etpro.rules.tar.gz + # Community rules from sourcefire. + community => { + summary => "Snort/VRT GPLv2 Community Rules", + website => "https://www.snort.ort", + tr_string => "community rules", + requires_subscription => "False", + dl_url => "https://www.snort.org/rules/community", + dl_type => "archive", + }, + # Emerging threads community rules. + emerging => { + summary => "Emergingthreats.net Community Rules", + website => "https://emergingtreads.net", + tr_string => "emerging rules", + requires_subscription => "False", + dl_url => "https://rules.emergingthreats.net/open/suricata-5.0/emerging.rules.tar.gz", + dl_type => "archive", + }, + + # Emerging threads Pro rules. + emerging_pro => { + summary => "Emergingthreats.net Pro Rules", + website => "https://emergingtreads.net", + tr_string => "emerging pro rules", + requires_subscription => "True", + dl_url => "https://rules.emergingthreatspro.com//suricata-5.0/etpro.rules.tar.gz", + dl_type => "archive", + }, +); From 179b75107e8a8bd0e339e3d5aef8372fafb9a0e9 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 19 Mar 2021 21:26:44 +0100 Subject: [PATCH 007/140] ids-functions.pl: Make downloader work with new ruleset-sources file format. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 0e397ca19..e9298ac4b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -169,10 +169,6 @@ sub downloadruleset { return 1; } - # Get all available ruleset locations. - my %rulesetsources=(); - &General::readhash($rulesetsourcesfile, \%rulesetsources); - # Read proxysettings. my %proxysettings=(); &General::readhash("${General::swroot}/proxy/settings", \%proxysettings); @@ -205,7 +201,7 @@ sub downloadruleset { } # Grab the right url based on the configured vendor. - my $url = $rulesetsources{$rulessettings{'RULES'}}; + my $url = $IDS::Ruleset::Providers{$rulessettings{'RULES'}}{'dl_url'}; # Check if the vendor requires an oinkcode and add it if needed. $url =~ s/\/$rulessettings{'OINKCODE'}/g; From 70cc13158d6d3be97d4094619cf20ec065023492 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 19 Mar 2021 21:27:23 +0100 Subject: [PATCH 008/140] ids-functions.pl: Add get_ruleset_providers() function. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index e9298ac4b..dd5257a45 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -112,6 +112,24 @@ sub check_and_create_filelayout() { unless (-f "$whitelist_file" ) { &create_empty_file($whitelist_file); } } +# +## Function to get a list of all available ruleset providers. +## +## They will be returned as a sorted array. +# +sub get_ruleset_providers() { + my @providers; + + # Loop through the hash of providers. + foreach my $provider ( keys %IDS::Ruleset::Providers ) { + # Add the provider to the array. + push(@providers, $provider); + } + + # Sort and return the array. + return sort(@providers); +} + # ## Function for checking if at least 300MB of free disk space are available ## on the "/var" partition. From 3e12c6e6883d80e4648aeed89a75c0c0de1d120b Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 19 Mar 2021 21:28:00 +0100 Subject: [PATCH 009/140] ids.cgi: Make CGI work with new ruleset-sources file. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 4e8b28fd8..04222d9f0 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -31,6 +31,9 @@ require "${General::swroot}/header.pl"; require "${General::swroot}/ids-functions.pl"; require "${General::swroot}/network-functions.pl"; +# Import ruleset providers file. +require "$IDS::rulesetsourcesfile"; + my %color = (); my %mainsettings = (); my %idsrules = (); @@ -324,17 +327,11 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Read-in current (old) IDS settings. &General::readhash("$IDS::rules_settings_file", \%oldsettings); - # Get all available ruleset locations. - &General::readhash("$IDS::rulesetsourcesfile", \%rulesetsources); - # Prevent form name from been stored in conf file. delete $cgiparams{'RULESET'}; - # Grab the URL based on the choosen vendor. - my $url = $rulesetsources{$cgiparams{'RULES'}}; - # Check if the choosen vendor (URL) requires an subscription/oinkcode. - if ($url =~ /\/ ) { + if ($IDS::Ruleset::Providers{$cgiparams{'RULES'}}{'requires_subscription'} eq "True") { # Check if an subscription/oinkcode has been provided. if ($cgiparams{'OINKCODE'}) { # Check if the oinkcode contains unallowed chars. @@ -689,10 +686,6 @@ $checked{'MONITOR_TRAFFIC_ONLY'}{'off'} = ''; $checked{'MONITOR_TRAFFIC_ONLY'}{'on'} = ''; $checked{'MONITOR_TRAFFIC_ONLY'}{$idssettings{'MONITOR_TRAFFIC_ONLY'}} = "checked='checked'"; $selected{'RULES'}{'nothing'} = ''; -$selected{'RULES'}{'community'} = ''; -$selected{'RULES'}{'emerging'} = ''; -$selected{'RULES'}{'registered'} = ''; -$selected{'RULES'}{'subscripted'} = ''; $selected{'RULES'}{$rulessettings{'RULES'}} = "selected='selected'"; $selected{'AUTOUPDATE_INTERVAL'}{'off'} = ''; $selected{'AUTOUPDATE_INTERVAL'}{'daily'} = ''; @@ -902,11 +895,30 @@ print < \n"; } print < diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index c81b28fea..9cc0a010e 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1375,6 +1375,7 @@ 'ids apply' => 'Übernehmen', 'ids apply ruleset changes' => 'Regeländerungen werden übernommen. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids automatic rules update' => 'Automatische Regelaktualisierung', +'ids customize ruleset' => 'Regelset anpassen', 'ids download new ruleset' => 'Das neue Regelsatz wird heruntergeladen und entpackt. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids enable' => 'Einbruchsverhinderungssystem aktivieren', 'ids hide' => 'Verstecken', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index a92bb07f8..3e0df8c0c 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1404,6 +1404,7 @@ 'ids apply' => 'Apply', 'ids apply ruleset changes' => 'The ruleset changes are being applied. Please wait until all operations have completed successfully...', 'ids automatic rules update' => 'Automatic Rule Update', +'ids customize ruleset' => 'Customize ruleset', 'ids download new ruleset' => 'Downloading and unpacking new ruleset. Please wait until all operations have completed successfully...', 'ids enable' => 'Enable Intrusion Prevention System', 'ids hide' => 'Hide', From a468b62b62d5a9f777fe1c4d4564ade7d70ed621 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 21 Mar 2021 17:22:29 +0100 Subject: [PATCH 015/140] ids.cgi: Only read-in ruleset if neccessary. This process takes some time, especially on huge rulesets. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 99 +++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index a2d650fb0..50a8709a7 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -250,69 +250,72 @@ if (-e $IDS::storederrorfile) { unlink($IDS::storederrorfile); } -## Grab all available rules and store them in the idsrules hash. -# -# Open rules directory and do a directory listing. -opendir(DIR, $IDS::rulespath) or die $!; - # Loop through the direcory. - while (my $file = readdir(DIR)) { +# Gather ruleset details. +if ($cgiparams{'RULESET'}) { + ## Grab all available rules and store them in the idsrules hash. + # + # Open rules directory and do a directory listing. + opendir(DIR, $IDS::rulespath) or die $!; + # Loop through the direcory. + while (my $file = readdir(DIR)) { - # We only want files. - next unless (-f "$IDS::rulespath/$file"); + # We only want files. + next unless (-f "$IDS::rulespath/$file"); - # Ignore empty files. - next if (-z "$IDS::rulespath/$file"); + # Ignore empty files. + next if (-z "$IDS::rulespath/$file"); - # Use a regular expression to find files ending in .rules - next unless ($file =~ m/\.rules$/); + # Use a regular expression to find files ending in .rules + next unless ($file =~ m/\.rules$/); - # Ignore files which are not read-able. - next unless (-R "$IDS::rulespath/$file"); + # Ignore files which are not read-able. + next unless (-R "$IDS::rulespath/$file"); - # Skip whitelist rules file. - next if( $file eq "whitelist.rules"); + # Skip whitelist rules file. + next if( $file eq "whitelist.rules"); - # Call subfunction to read-in rulefile and add rules to - # the idsrules hash. - &readrulesfile("$file"); - } + # Call subfunction to read-in rulefile and add rules to + # the idsrules hash. + &readrulesfile("$file"); + } -closedir(DIR); + closedir(DIR); -# Gather used rulefiles. -# -# Check if the file for activated rulefiles is not empty. -if(-f $IDS::used_rulefiles_file) { - # Open the file for used rulefile and read-in content. - open(FILE, $IDS::used_rulefiles_file) or die "Could not open $IDS::used_rulefiles_file. $!\n"; + # Gather used rulefiles. + # + # Check if the file for activated rulefiles is not empty. + if(-f $IDS::used_rulefiles_file) { + # Open the file for used rulefile and read-in content. + open(FILE, $IDS::used_rulefiles_file) or die "Could not open $IDS::used_rulefiles_file. $!\n"; - # Read-in content. - my @lines = ; + # Read-in content. + my @lines = ; - # Close file. - close(FILE); + # Close file. + close(FILE); - # Loop through the array. - foreach my $line (@lines) { - # Remove newlines. - chomp($line); + # Loop through the array. + foreach my $line (@lines) { + # Remove newlines. + chomp($line); - # Skip comments. - next if ($line =~ /\#/); + # Skip comments. + next if ($line =~ /\#/); - # Skip blank lines. - next if ($line =~ /^\s*$/); + # Skip blank lines. + next if ($line =~ /^\s*$/); - # Gather rule sid and message from the ruleline. - if ($line =~ /.*- (.*)/) { - my $rulefile = $1; + # Gather rule sid and message from the ruleline. + if ($line =~ /.*- (.*)/) { + my $rulefile = $1; - # Check if the current rulefile exists in the %idsrules hash. - # If not, the file probably does not exist anymore or contains - # no rules. - if($idsrules{$rulefile}) { - # Add the rulefile state to the %idsrules hash. - $idsrules{$rulefile}{'Rulefile'}{'State'} = "on"; + # Check if the current rulefile exists in the %idsrules hash. + # If not, the file probably does not exist anymore or contains + # no rules. + if($idsrules{$rulefile}) { + # Add the rulefile state to the %idsrules hash. + $idsrules{$rulefile}{'Rulefile'}{'State'} = "on"; + } } } } From 2bbe6ede23af14e0a23b77e1aaca8c5a26ecb6ac Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 21 Mar 2021 21:41:42 +0100 Subject: [PATCH 016/140] ids.cgi: Move / Splitt main page and customize ruleset subpage. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 632 ++++++++++++++++++++++--------------------- 1 file changed, 325 insertions(+), 307 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 50a8709a7..6af99e8e6 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -667,299 +667,299 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { } } -# Read-in idssettings and rulesetsettings -&General::readhash("$IDS::ids_settings_file", \%idssettings); -&General::readhash("$IDS::rules_settings_file", \%rulessettings); - -# If no autoupdate intervall has been configured yet, set default value. -unless(exists($rulessettings{'AUTOUPDATE_INTERVAL'})) { - # Set default to "weekly". - $rulessettings{'AUTOUPDATE_INTERVAL'} = 'weekly'; -} - -# Read-in ignored hosts. -&General::readhasharray("$IDS::settingsdir/ignored", \%ignored); - -$checked{'ENABLE_IDS'}{'off'} = ''; -$checked{'ENABLE_IDS'}{'on'} = ''; -$checked{'ENABLE_IDS'}{$idssettings{'ENABLE_IDS'}} = "checked='checked'"; -$checked{'MONITOR_TRAFFIC_ONLY'}{'off'} = ''; -$checked{'MONITOR_TRAFFIC_ONLY'}{'on'} = ''; -$checked{'MONITOR_TRAFFIC_ONLY'}{$idssettings{'MONITOR_TRAFFIC_ONLY'}} = "checked='checked'"; -$selected{'RULES'}{'nothing'} = ''; -$selected{'RULES'}{$rulessettings{'RULES'}} = "selected='selected'"; -$selected{'AUTOUPDATE_INTERVAL'}{'off'} = ''; -$selected{'AUTOUPDATE_INTERVAL'}{'daily'} = ''; -$selected{'AUTOUPDATE_INTERVAL'}{'weekly'} = ''; -$selected{'AUTOUPDATE_INTERVAL'}{$rulessettings{'AUTOUPDATE_INTERVAL'}} = "selected='selected'"; - &Header::openpage($Lang::tr{'intrusion detection system'}, 1, ''); -### Java Script ### -print" -END -; - &Header::openbigbox('100%', 'left', '', $errormessage); -if ($errormessage) { - &Header::openbox('100%', 'left', $Lang::tr{'error messages'}); - print "$errormessage\n"; - print " \n"; - &Header::closebox(); -} +&show_display_error_message(); -# Draw current state of the IDS -&Header::openbox('100%', 'left', $Lang::tr{'intrusion detection system'}); - -# Check if the IDS is running and obtain the process-id. -my $pid = &IDS::ids_is_running(); - -# Display some useful information, if suricata daemon is running. -if ($pid) { - # Gather used memory. - my $memory = &get_memory_usage($pid); - - print < - - $Lang::tr{'intrusion detection'} - - - - $Lang::tr{'guardian daemon'} - $Lang::tr{'running'} - - - - - PID - $Lang::tr{'memory'} - - - - - $pid - $memory KB - - -END +if ($cgiparams{'RULESET'} eq "$Lang::tr{'ids customize ruleset'}" ) { + &show_customize_ruleset(); } else { - # Otherwise display a hint that the service is not launched. - print < - - $Lang::tr{'intrusion detection'} - - - - $Lang::tr{'guardian daemon'} - $Lang::tr{'stopped'} - - -END + &show_mainpage(); } -# Only show this area, if a ruleset is present. -if (%idsrules) { +&Header::closebigbox(); +&Header::closepage(); - print <$errormessage\n"; + print " \n"; + &Header::closebox(); + } +} -

$Lang::tr{'settings'}

+# +## Function to display the main IDS page. +# +sub show_mainpage() { + # Read-in idssettings and rulesetsettings + &General::readhash("$IDS::ids_settings_file", \%idssettings); + &General::readhash("$IDS::rules_settings_file", \%rulessettings); -
- - - + # If no autoupdate intervall has been configured yet, set default value. + unless(exists($rulessettings{'AUTOUPDATE_INTERVAL'})) { + # Set default to "weekly". + $rulessettings{'AUTOUPDATE_INTERVAL'} = 'weekly'; + } - - + # Read-in ignored hosts. + &General::readhasharray("$IDS::settingsdir/ignored", \%ignored); - - - - - - + $checked{'ENABLE_IDS'}{'off'} = ''; + $checked{'ENABLE_IDS'}{'on'} = ''; + $checked{'ENABLE_IDS'}{$idssettings{'ENABLE_IDS'}} = "checked='checked'"; + $checked{'MONITOR_TRAFFIC_ONLY'}{'off'} = ''; + $checked{'MONITOR_TRAFFIC_ONLY'}{'on'} = ''; + $checked{'MONITOR_TRAFFIC_ONLY'}{$idssettings{'MONITOR_TRAFFIC_ONLY'}} = "checked='checked'"; + $selected{'RULES'}{'nothing'} = ''; + $selected{'RULES'}{$rulessettings{'RULES'}} = "selected='selected'"; + $selected{'AUTOUPDATE_INTERVAL'}{'off'} = ''; + $selected{'AUTOUPDATE_INTERVAL'}{'daily'} = ''; + $selected{'AUTOUPDATE_INTERVAL'}{'weekly'} = ''; + $selected{'AUTOUPDATE_INTERVAL'}{$rulessettings{'AUTOUPDATE_INTERVAL'}} = "selected='selected'"; - - - + ### Java Script ### + print " END ; - # Loop through the array of available networks and print config options. - foreach my $zone (@network_zones) { - my $checked_input; - my $checked_forward; + # Draw current state of the IDS + &Header::openbox('100%', 'left', $Lang::tr{'intrusion detection system'}); - # Convert current zone name to upper case. - my $zone_upper = uc($zone); + # Check if the IDS is running and obtain the process-id. + my $pid = &IDS::ids_is_running(); - # Set zone name. - my $zone_name = $zone; + # Display some useful information, if suricata daemon is running. + if ($pid) { + # Gather used memory. + my $memory = &get_memory_usage($pid); - # Dirty hack to get the correct language string for the red zone. - if ($zone eq "red") { - $zone_name = "red1"; - } + print < + + + - # Grab checkbox status from settings hash. - if ($idssettings{"ENABLE_IDS_$zone_upper"} eq "on") { - $checked_input = "checked = 'checked'"; - } + + + + - print "\n"; + + + + + + + + + + + +
-  $Lang::tr{'ids enable'} - -  $Lang::tr{'ids monitor traffic only'} -








$Lang::tr{'ids monitored interfaces'}
$Lang::tr{'intrusion detection'}
$Lang::tr{'guardian daemon'}$Lang::tr{'running'}
\n"; - print "\n"; - print " $Lang::tr{'enabled on'} $Lang::tr{$zone_name}\n"; - print "
PID$Lang::tr{'memory'}
$pid$memory KB
+END + } else { + # Otherwise display a hint that the service is not launched. + print < + + $Lang::tr{'intrusion detection'} + + + + $Lang::tr{'guardian daemon'} + $Lang::tr{'stopped'} + + +END } + # Only show this area, if a ruleset is present. + if (%idsrules) { + print < - -

+

$Lang::tr{'settings'}

- + +
+ + + + + + + + + + + + + + + + + + +END +; + + # Loop through the array of available networks and print config options. + foreach my $zone (@network_zones) { + my $checked_input; + my $checked_forward; + + # Convert current zone name to upper case. + my $zone_upper = uc($zone); + + # Set zone name. + my $zone_name = $zone; + + # Dirty hack to get the correct language string for the red zone. + if ($zone eq "red") { + $zone_name = "red1"; + } + + # Grab checkbox status from settings hash. + if ($idssettings{"ENABLE_IDS_$zone_upper"} eq "on") { + $checked_input = "checked = 'checked'"; + } + + print "\n"; + } + +print < +
+  $Lang::tr{'ids enable'} + +  $Lang::tr{'ids monitor traffic only'} +








$Lang::tr{'ids monitored interfaces'}
\n"; + print "\n"; + print " $Lang::tr{'enabled on'} $Lang::tr{$zone_name}\n"; + print "
+ +

+ + + + + +
+ +END +; + + } + + &Header::closebox(); + + # Draw elements for ruleset configuration. + &Header::openbox('100%', 'center', $Lang::tr{'ids ruleset settings'}); + +print < + - + + + + + + + + + + + + + + + + + + + + + + +
$Lang::tr{'ids rules update'}$Lang::tr{'ids automatic rules update'}
+ +


  +END +; + # Show the "Update Ruleset"-Button only if a ruleset has been downloaded yet and automatic updates are disabled. + if ((%idsrules) && ($rulessettings{'AUTOUPDATE_INTERVAL'} eq "off")) { + # Display button to update the ruleset. + print"\n"; + } +print < + +
END ; -} + &Header::closebox(); -&Header::closebox(); + # + # Whitelist / Ignorelist + # + &Header::openbox('100%', 'center', $Lang::tr{'ids ignored hosts'}); -# Draw elements for ruleset configuration. -&Header::openbox('100%', 'center', $Lang::tr{'ids ruleset settings'}); - -print < - - - - - - - - - - - - - - - - - - - - - - - - - - -
$Lang::tr{'ids rules update'}$Lang::tr{'ids automatic rules update'}
- -


  -END -; - # Show the "Update Ruleset"-Button only if a ruleset has been downloaded yet and automatic updates are disabled. - if ((%idsrules) && ($rulessettings{'AUTOUPDATE_INTERVAL'} eq "off")) { - # Display button to update the ruleset. - print"\n"; - } -print < - -
- -END -; - -&Header::closebox(); - -# -# Whitelist / Ignorelist -# -&Header::openbox('100%', 'center', $Lang::tr{'ids ignored hosts'}); - -print < $Lang::tr{'ip address'} @@ -1027,77 +1027,95 @@ print < - - + + END + } + } else { + # Print notice that currently no hosts are ignored. + print "\n"; + print "$Lang::tr{'guardian no entries'}\n"; + print "\n"; } - } else { - # Print notice that currently no hosts are ignored. - print "\n"; - print "$Lang::tr{'guardian no entries'}\n"; - print "\n"; - } - print "\n"; + print "\n"; - # Section to add new elements or edit existing ones. + # Section to add new elements or edit existing ones. print < -
-
- -
- +
+
+
+ +
+
END - # Assign correct headline and button text. - my $buttontext; - my $entry_address; - my $entry_remark; + # Assign correct headline and button text. + my $buttontext; + my $entry_address; + my $entry_remark; - # Check if an ID (key) has been given, in this case an existing entry should be edited. - if ($cgiparams{'ID'} ne '') { - $buttontext = $Lang::tr{'update'}; - print "\n"; + # Check if an ID (key) has been given, in this case an existing entry should be edited. + if ($cgiparams{'ID'} ne '') { + $buttontext = $Lang::tr{'update'}; + print "\n"; - # Grab address and remark for the given key. - $entry_address = $ignored{$cgiparams{'ID'}}[0]; - $entry_remark = $ignored{$cgiparams{'ID'}}[1]; - } else { - $buttontext = $Lang::tr{'add'}; - print "\n"; - } + # Grab address and remark for the given key. + $entry_address = $ignored{$cgiparams{'ID'}}[0]; + $entry_remark = $ignored{$cgiparams{'ID'}}[1]; + } else { + $buttontext = $Lang::tr{'add'}; + print "\n"; + } print < - - - - + + + + + - - - - - -
$Lang::tr{'update'}
$Lang::tr{'update'}
$Lang::tr{'dnsforward add a new entry'}
$Lang::tr{'dnsforward add a new entry'}
$Lang::tr{'ip address'}:
$Lang::tr{'ip address'}: $Lang::tr{'remark'}:
-
+ $Lang::tr{'remark'}: + + + + + + END -&Header::closebox(); - -# Only show the section for configuring the ruleset if one is present. -if (%idsrules) { - &show_customize_ruleset(); + &Header::closebox(); } -&Header::closebigbox(); -&Header::closepage(); - # ## Function to show the customize ruleset section. # sub show_customize_ruleset() { + ### Java Script ### + print" +END +; # Load neccessary perl modules for file stat and to format the timestamp. use File::stat; use POSIX qw( strftime ); From 4efc8ccd8aacedbc1c24908fa7ee3989182ba976 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 22 Mar 2021 11:48:58 +0100 Subject: [PATCH 017/140] ids.cgi: Add "Back" button to customize ruleset sub-page. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 6af99e8e6..01fb96cbe 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1225,7 +1225,10 @@ END print < - + + + + From 87df37da7abbb04eb83d0803c1237039c8dddfd3 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 22 Mar 2021 14:42:42 +0100 Subject: [PATCH 018/140] ids.cgi: Stop showing ruleset date on customize rulest sub-page. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 01fb96cbe..3f0ab07b2 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1116,22 +1116,7 @@ print < END ; - # Load neccessary perl modules for file stat and to format the timestamp. - use File::stat; - use POSIX qw( strftime ); - - # Call stat on the rulestarball. - my $stat = stat("$IDS::rulestarball"); - - if (defined $stat) { - # Get timestamp the file creation. - my $mtime = $stat->mtime; - - # Convert into human read-able format. - my $rulesdate = strftime('%Y-%m-%d %H:%M:%S', localtime($mtime)); - - &Header::openbox('100%', 'LEFT', "$Lang::tr{'intrusion detection system rules'} ($rulesdate)" ); - + &Header::openbox('100%', 'LEFT', "$Lang::tr{'intrusion detection system rules'}" ); print"
\n"; # Output display table for rule files From 77351a6b768c0359cb0a25d540f5f2c1ff03e51f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 22 Mar 2021 16:27:33 +0100 Subject: [PATCH 019/140] ids.cgi: Move configuration of ruleset autoupdate intervall to IDS main section. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 3f0ab07b2..42d2ca7fb 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -351,12 +351,6 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Store settings into settings file. &General::writehash("$IDS::rules_settings_file", \%cgiparams); - # Check if the the automatic rule update hass been touched. - if($cgiparams{'AUTOUPDATE_INTERVAL'} ne $oldsettings{'AUTOUPDATE_INTERVAL'}) { - # Call suricatactrl to set the new interval. - &IDS::call_suricatactrl("cron", $cgiparams{'AUTOUPDATE_INTERVAL'}); - } - # Check if a ruleset is present - if not or the source has been changed download it. if((! %idsrules) || ($oldsettings{'RULES'} ne $cgiparams{'RULES'})) { # Check if the red device is active. @@ -618,6 +612,12 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { &General::writehash("$IDS::ids_settings_file", \%cgiparams); } + # Check if the the automatic rule update hass been touched. + if($cgiparams{'AUTOUPDATE_INTERVAL'} ne $oldidssettings{'AUTOUPDATE_INTERVAL'}) { + # Call suricatactrl to set the new interval. + &IDS::call_suricatactrl("cron", $cgiparams{'AUTOUPDATE_INTERVAL'}); + } + # Generate file to store the home net. &IDS::generate_home_net_file(); @@ -703,9 +703,9 @@ sub show_mainpage() { &General::readhash("$IDS::rules_settings_file", \%rulessettings); # If no autoupdate intervall has been configured yet, set default value. - unless(exists($rulessettings{'AUTOUPDATE_INTERVAL'})) { + unless(exists($idssettings{'AUTOUPDATE_INTERVAL'})) { # Set default to "weekly". - $rulessettings{'AUTOUPDATE_INTERVAL'} = 'weekly'; + $idssettings{'AUTOUPDATE_INTERVAL'} = 'weekly'; } # Read-in ignored hosts. @@ -866,6 +866,27 @@ END print < + + +

+

+

+

+ + + + $Lang::tr{'ids automatic rules update'} + + + + + + +

From a49a30d1ba7560c9e394570f4d313a575820c439 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 12:39:13 +0100 Subject: [PATCH 020/140] ruleset-sources: Fix website details for emergingthreats provider. Signed-off-by: Stefan Schantl --- config/suricata/ruleset-sources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index 638d32fb8..a4f556118 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -50,7 +50,7 @@ our %Providers = ( # Emerging threads community rules. emerging => { summary => "Emergingthreats.net Community Rules", - website => "https://emergingtreads.net", + website => "https://emergingthreats.net/", tr_string => "emerging rules", requires_subscription => "False", dl_url => "https://rules.emergingthreats.net/open/suricata-5.0/emerging.rules.tar.gz", @@ -60,7 +60,7 @@ our %Providers = ( # Emerging threads Pro rules. emerging_pro => { summary => "Emergingthreats.net Pro Rules", - website => "https://emergingtreads.net", + website => "https://emergingthreats.net/", tr_string => "emerging pro rules", requires_subscription => "True", dl_url => "https://rules.emergingthreatspro.com//suricata-5.0/etpro.rules.tar.gz", From 2f252efa0dc0625248ecf99917be9887821b5223 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 14:56:11 +0100 Subject: [PATCH 021/140] ids.cgi: Rework rulesetsettings section. * The page and section now supports multiple ruleset providers at once. * Adding / Editing a ruleset provider has been moved to a own sub-page. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 428 +++++++++++++++++++++++++++++++++---------- 1 file changed, 331 insertions(+), 97 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 42d2ca7fb..e922e5dd3 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -38,7 +38,7 @@ my %color = (); my %mainsettings = (); my %idsrules = (); my %idssettings=(); -my %rulessettings=(); +my %used_providers=(); my %cgiparams=(); my %checked=(); my %selected=(); @@ -52,6 +52,9 @@ my %ignored=(); # the list of zones in an array. my @network_zones = &Network::get_available_network_zones(); +# Grab all used ruleset providers. +&General::readhasharray($IDS::providers_settings_file, \%used_providers); + # Check if openvpn is started and add it to the array of network zones. if ( -e "/var/run/openvpn.pid") { push(@network_zones, "ovpn"); @@ -584,7 +587,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Check if the IDS should be enabled. if ($cgiparams{'ENABLE_IDS'} eq "on") { # Check if any ruleset is available. Otherwise abort and display an error. - unless(%idsrules) { + unless(%used_providers) { $errormessage = $Lang::tr{'ids no ruleset available'}; } @@ -633,7 +636,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Check if "MONITOR_TRAFFIC_ONLY" has been changed. if($cgiparams{'MONITOR_TRAFFIC_ONLY'} ne $oldidssettings{'MONITOR_TRAFFIC_ONLY'}) { # Check if a ruleset exists. - if (%idsrules) { + if (%used_providers) { # Lock the webpage and print message. &working_notice("$Lang::tr{'ids working'}"); @@ -675,6 +678,8 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { if ($cgiparams{'RULESET'} eq "$Lang::tr{'ids customize ruleset'}" ) { &show_customize_ruleset(); +} elsif ($cgiparams{'PROVIDERS'} ne "") { + &show_add_provider(); } else { &show_mainpage(); } @@ -698,9 +703,8 @@ sub show_display_error_message() { ## Function to display the main IDS page. # sub show_mainpage() { - # Read-in idssettings and rulesetsettings + # Read-in idssettings . &General::readhash("$IDS::ids_settings_file", \%idssettings); - &General::readhash("$IDS::rules_settings_file", \%rulessettings); # If no autoupdate intervall has been configured yet, set default value. unless(exists($idssettings{'AUTOUPDATE_INTERVAL'})) { @@ -717,40 +721,10 @@ sub show_mainpage() { $checked{'MONITOR_TRAFFIC_ONLY'}{'off'} = ''; $checked{'MONITOR_TRAFFIC_ONLY'}{'on'} = ''; $checked{'MONITOR_TRAFFIC_ONLY'}{$idssettings{'MONITOR_TRAFFIC_ONLY'}} = "checked='checked'"; - $selected{'RULES'}{'nothing'} = ''; - $selected{'RULES'}{$rulessettings{'RULES'}} = "selected='selected'"; $selected{'AUTOUPDATE_INTERVAL'}{'off'} = ''; $selected{'AUTOUPDATE_INTERVAL'}{'daily'} = ''; $selected{'AUTOUPDATE_INTERVAL'}{'weekly'} = ''; - $selected{'AUTOUPDATE_INTERVAL'}{$rulessettings{'AUTOUPDATE_INTERVAL'}} = "selected='selected'"; - - ### Java Script ### - print " -END -; + $selected{'AUTOUPDATE_INTERVAL'}{$idssettings{'AUTOUPDATE_INTERVAL'}} = "selected='selected'"; # Draw current state of the IDS &Header::openbox('100%', 'left', $Lang::tr{'intrusion detection system'}); @@ -803,8 +777,8 @@ END END } - # Only show this area, if a ruleset is present. - if (%idsrules) { + # Only show this area, if at least one ruleset provider is configured. + if (%used_providers) { print < - - - - - - - - + + + + + + END -; - # Get all available ruleset providers. - my @ruleset_providers = &IDS::get_ruleset_providers(); + # Check if some providers has been configured. + if (keys (%used_providers)) { + my $col = ""; - # Loop throgh the list of providers. - foreach my $provider (@ruleset_providers) { - # Call get_provider_name function to obtain the provider name. - my $option_string = &get_provider_name($provider); + # Loop through all entries of the hash. + #while( (my $id) = each %used_providers) { + foreach my $id (sort keys(%used_providers)) { + # Assign data array positions to some nice variable names. + my $provider = $used_providers{$id}[0]; + my $provider_name = &get_provider_name($provider); - # Print option. - print "\n"; + #XXX my $rulesdate = &IDS::get_ruleset_date($provider); + my $rulesdate; + + my $subscription_code = $used_providers{$id}[1]; + my $autoupdate_status = $used_providers{$id}[2]; + my $status = $used_providers{$id}[3]; + + # Check if the item number is even or not. + if ($id % 2) { + $col="bgcolor='$color{'color22'}'"; + } else { + $col="bgcolor='$color{'color20'}'"; } + + # Choose icons for the checkboxes. + my $status_gif; + my $status_gdesc; + my $autoupdate_status_gif; + my $autoupdate_status_gdesc; + + # Check if the status is enabled and select the correct image and description. + if ($status eq 'enabled' ) { + $status_gif = 'on.gif'; + $status_gdesc = $Lang::tr{'click to disable'}; + } else { + $status_gif = 'off.gif'; + $status_gdesc = $Lang::tr{'click to enable'}; + } + + # Check if the autoupdate status is enabled and select the correct image and description. + if ($autoupdate_status eq 'enabled') { + $autoupdate_status_gif = 'on.gif'; + $autoupdate_status_gdesc = $Lang::tr{'click to disable'}; + } else { + $autoupdate_status_gif = 'off.gif'; + $autoupdate_status_gdesc = $Lang::tr{'click to enable'}; + } + print < - + + + - - + - - - + - - - + - - - - + END -; - # Show the "Update Ruleset"-Button only if a ruleset has been downloaded yet and automatic updates are disabled. - if ((%idsrules) && ($rulessettings{'AUTOUPDATE_INTERVAL'} eq "off")) { - # Display button to update the ruleset. - print"\n"; } -print < - - - -
$Lang::tr{'ids rules update'}$Lang::tr{'ids automatic rules update'}
$Lang::tr{'ids provider'}$Lang::tr{'date'}$Lang::tr{'ids autoupdates'}
$provider_name$rulesdate - -
+ + + + + +


+
+ + + +
+
+
+ + + +
+
  + +
+ + + +
+
- + } else { + # Print notice that currently no hosts are ignored. + print "\n"; + print "$Lang::tr{'guardian no entries'}\n"; + print "\n"; + } + + print "\n"; + + # Section to add new elements or edit existing ones. +print < +
+
+ +
+ + + +END + + # Only show this button if a ruleset provider is configured. + if (%used_providers) { + print "\n"; + } +print < + + +
+
END -; &Header::closebox(); @@ -1028,17 +1070,17 @@ print <
- - - + + +
- - - + + +
@@ -1244,6 +1286,198 @@ END } } +# +## Function to show section for add/edit a provider. +# +sub show_add_provider() { + my @subscription_providers; + + # Get all supported ruleset providers. + my @ruleset_providers = &IDS::get_ruleset_providers(); + + ### Java Script ### + print " +END +; + + &Header::openbox('100%', 'center', $Lang::tr{'ids provider settings'}); + + # Check if an existing provider should be edited. + if($cgiparams{'PROVIDERS'} eq "$Lang::tr{'edit'}") { + # Check if autoupdate is enabled for this provider. + if ($used_providers{$cgiparams{'ID'}}[2] eq "on") { + # Set the checkbox to be checked. + $checked{'ENABLE_AUTOUPDATE'} = "checked='checked'"; + } + } elsif ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'ids add provider'}") { + # Set the autoupdate to true as default. + $checked{'ENABLE_AUTOUPDATE'} = "checked='checked'"; + } + +print < + + + + + + + + + + + + + + + + + + + + + + + + + +
$Lang::tr{'ids provider'}
+ + + $Lang::tr{'ids visit provider website'} +


+  $Lang::tr{'ids enable automatic updates'} +
+ +END +; + # Check if a provider should be added or edited. + if ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'edit'}") { + # Display button for updating the existing provider. + print "\n"; + } else { + # Display button to add the new provider. + print "\n"; + } +print < +
+ +END +; + &Header::closebox(); +} + # ## A function to display a notice, to lock the webpage and ## tell the user which action currently will be performed. From a8d36d3e1fe619309b047aabce38b1e105f692c8 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 14:58:09 +0100 Subject: [PATCH 022/140] ids-functions.pl: Introduce providers_settings_file. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 347257070..3e146b589 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -54,9 +54,12 @@ our $modify_sids_file = "$settingsdir/oinkmaster-modify-sids.conf"; # File which stores the configured IPS settings. our $ids_settings_file = "$settingsdir/settings"; -# File which stores the configured rules-settings. +# DEPRECATED - File which stores the configured rules-settings. our $rules_settings_file = "$settingsdir/rules-settings"; +# File which stores the used and configured ruleset providers. +our $providers_settings_file = "$settingsdir/providers-settings"; + # File which stores the configured settings for whitelisted addresses. our $ignored_file = "$settingsdir/ignored"; @@ -108,7 +111,7 @@ sub check_and_create_filelayout() { unless (-f "$modify_sids_file") { &create_empty_file($modify_sids_file); } unless (-f "$used_rulefiles_file") { &create_empty_file($used_rulefiles_file); } unless (-f "$ids_settings_file") { &create_empty_file($ids_settings_file); } - unless (-f "$rules_settings_file") { &create_empty_file($rules_settings_file); } + unless (-f "$providers_settings_file") { &create_empty_file($providers_settings_file); } unless (-f "$ignored_file") { &create_empty_file($ignored_file); } unless (-f "$whitelist_file" ) { &create_empty_file($whitelist_file); } } From 18fb2dbd5c17edb58eaba1f6a609bab798795c63 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 16:20:17 +0100 Subject: [PATCH 023/140] Update language files. Signed-off-by: Stefan Schantl --- langs/de/cgi-bin/de.pl | 5 +++++ langs/en/cgi-bin/en.pl | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 9cc0a010e..3a4a52e5f 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1372,12 +1372,14 @@ 'idle' => 'Leerlauf', 'idle timeout' => 'Leerlaufwartezeit in Minuten (0 zum Deaktivieren):', 'idle timeout not set' => 'Leerlaufwartezeit nicht angegeben.', +'ids add provider' => 'Provider hinzufügen', 'ids apply' => 'Übernehmen', 'ids apply ruleset changes' => 'Regeländerungen werden übernommen. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids automatic rules update' => 'Automatische Regelaktualisierung', 'ids customize ruleset' => 'Regelset anpassen', 'ids download new ruleset' => 'Das neue Regelsatz wird heruntergeladen und entpackt. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids enable' => 'Einbruchsverhinderungssystem aktivieren', +'ids enable automatic updates' => 'Automatische Updates aktivieren', 'ids hide' => 'Verstecken', 'ids ignored hosts' => 'Ausnahmeliste', 'ids log hits' => 'Gesamtanzahl der Regeltreffer für', @@ -1388,10 +1390,13 @@ 'ids no network zone' => 'Bitte wählen Sie mindestens eine zu überwachende Netzwerkzone aus', 'ids no ruleset available' => 'Es ist kein Regelsatz verfügbar. Bitte laden Sie einen Regelsatz herunter.', 'ids oinkcode required' => 'Für den ausgewählten Regelsatz wird ein Abonnement oder ein Oinkcode benötigt', +'ids provider' => 'Regelset-Anbieter', +'ids provider settings' => 'Regelset-Anbieter-Einstellungen', 'ids rules update' => 'Regelsatz', 'ids ruleset autoupdate in progress' => 'Der Regelsatz wird gerade aktualisiert. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids ruleset settings' => 'Regelsatzeinstellungen', 'ids show' => 'Anzeigen', +'ids visit provider website' => 'Anbieter-Webseite besuchen', 'ids working' => 'Änderungen werden übernommen. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde.', 'iface' => 'Iface', 'ignore filter' => '"Ignorieren"-Filter', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index 3e0df8c0c..a62ef0382 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1401,12 +1401,15 @@ 'idle' => 'Idle', 'idle timeout' => 'Idle timeout (mins; 0 to disable):', 'idle timeout not set' => 'Idle timeout not set.', +'ids add provider' => 'Add provider', 'ids apply' => 'Apply', 'ids apply ruleset changes' => 'The ruleset changes are being applied. Please wait until all operations have completed successfully...', +'ids autoupdates' => 'Automatic updates', 'ids automatic rules update' => 'Automatic Rule Update', 'ids customize ruleset' => 'Customize ruleset', 'ids download new ruleset' => 'Downloading and unpacking new ruleset. Please wait until all operations have completed successfully...', 'ids enable' => 'Enable Intrusion Prevention System', +'ids enable automatic updates' => 'Enable automatic updates', 'ids hide' => 'Hide', 'ids ignored hosts' => 'Whitelisted Hosts', 'ids log hits' => 'Total of number of activated rules for', @@ -1416,11 +1419,14 @@ 'ids monitored interfaces' => 'Monitored Interfaces', 'ids no network zone' => 'Please select at least one network zone to be monitored', 'ids no ruleset available' => 'No ruleset is available. Please download one first', -'ids oinkcode required' => 'The selected ruleset requires a subscription or an Oinkcode', +'ids subscription code required' => 'The selected ruleset requires a subscription code', +'ids provider' => 'Provider', +'ids provider settings' => 'Provider settings', 'ids rules update' => 'Ruleset', 'ids ruleset autoupdate in progress' => 'Ruleset update in progress. Please wait until all operations have completed successfully...', 'ids ruleset settings' => 'Ruleset Settings', 'ids show' => 'Show', +'ids visit provider website' => 'Visit provider website', 'ids working' => 'Changes are being applied. Please wait until all operations have completed successfully...', 'iface' => 'Iface', 'ignore filter' => 'Ignore filter', @@ -1498,7 +1504,7 @@ 'invalid input for max clients' => 'Invalid input for Max Clients. The maximum of 1024 clients has been exceeded', 'invalid input for mode' => 'Invalid input for mode', 'invalid input for name' => 'Invalid input for user\'s full name or system hostname', -'invalid input for oink code' => 'Invalid input for Oink code', +'invalid input for subscription code' => 'Invalid input for subscription code', 'invalid input for organization' => 'Invalid input for organization', 'invalid input for remote host/ip' => 'Invalid input for remote host/ip.', 'invalid input for state or province' => 'Invalid input for state or province.', @@ -2372,6 +2378,7 @@ 'subnet is invalid' => 'Netmask is invalid', 'subnet mask' => 'Subnet Mask', 'subscripted user rules' => 'Talos VRT rules with subscription', +'subscription code' => 'Subscription code', 'successfully refreshed updates list' => 'Successfully refreshed updates list.', 'summaries kept' => 'Keep summaries for', 'sunday' => 'Sunday', From 4c067847c5db9d96aa7d6f1ef613b60f211817f9 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 16:20:52 +0100 Subject: [PATCH 024/140] ids.cgi: Add code to add/edit a ruleset provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index e922e5dd3..739e48eaa 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -668,6 +668,84 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Perform a reload of the page. &reload(); } + +} elsif (($cgiparams{'PROVIDERS'} eq "$Lang::tr{'add'}") || ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'update'}")) { + # Assign some nice human-readable values. + my $provider = $cgiparams{'PROVIDER'}; + my $subscription_code = $cgiparams{'SUBSCRIPTION_CODE'}; + my $status_autoupdate = $cgiparams{'ENABLE_AUTOUPDATE'}; + + # Check if we are going to add a new provider. + if ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'add'}") { + # Loop through the hash of used providers. + foreach my $id ( keys %used_providers) { + # Check if the choosen provider is already in use. + if ($used_providers{$id}[0] eq "$provider") { + # XXX - add to language file. + # Assign error message. + $errormessage = "The choosen provider is already in use."; + } + } + } + + # Check if the provider requires a subscription code. + if ($IDS::Ruleset::Providers{$provider}{'requires_subscription'} eq "True") { + # Check if an subscription code has been provided. + if ($subscription_code) { + # Check if the code contains unallowed chars. + unless ($subscription_code =~ /^[a-z0-9]+$/) { + $errormessage = $Lang::tr{'invalid input for subscription code'}; + } + } else { + # Print an error message, that an subsription code is required for this + # provider. + $errormessage = $Lang::tr{'ids subscription code required'}; + } + } + + # Go further if there was no error. + if ($errormessage eq '') { + my $id; + my $status; + + # Check if we should edit an existing entry and got an ID. + if (($cgiparams{'PROVIDERS'} eq $Lang::tr{'update'}) && ($cgiparams{'ID'})) { + # Assin the provided id. + $id = $cgiparams{'ID'}; + + # Undef the given ID. + undef($cgiparams{'ID'}); + + # Grab the configured status of the corresponding entry. + $status = $used_providers{$id}[3]; + } else { + # Each newly added entry automatically should be enabled. + $status = "enabled"; + + # Generate the ID for the new entry. + # + # Sort the keys by their ID and store them in an array. + my @keys = sort { $a <=> $b } keys %used_providers; + + # Reverse the key array. + my @reversed = reverse(@keys); + + # Obtain the last used id. + my $last_id = @reversed[0]; + + # Increase the last id by one and use it as id for the new entry. + $id = ++$last_id; + } + + # Add/Modify the entry to/in the used providers hash.. + $used_providers{$id} = ["$provider", "$subscription_code", "$status_autoupdate", "$status"]; + + # Write the changed hash to the providers settings file. + &General::writehasharray($IDS::providers_settings_file, \%used_providers); + + # Undefine providers flag. + undef($cgiparams{'PROVIDERS'}); + } } &Header::openpage($Lang::tr{'intrusion detection system'}, 1, ''); From aba3cbe5bc0278132230bf027e54349f1a89b3d6 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 17:37:33 +0100 Subject: [PATCH 025/140] ids.cgi: Read-in providers settings file when neccessary. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 739e48eaa..668dc7bcb 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -52,9 +52,6 @@ my %ignored=(); # the list of zones in an array. my @network_zones = &Network::get_available_network_zones(); -# Grab all used ruleset providers. -&General::readhasharray($IDS::providers_settings_file, \%used_providers); - # Check if openvpn is started and add it to the array of network zones. if ( -e "/var/run/openvpn.pid") { push(@network_zones, "ovpn"); @@ -670,6 +667,11 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { } } elsif (($cgiparams{'PROVIDERS'} eq "$Lang::tr{'add'}") || ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'update'}")) { + my %used_providers = (); + + # Read-in providers settings file. + &General::readhasharray("$IDS::providers_settings_file", \%used_providers); + # Assign some nice human-readable values. my $provider = $cgiparams{'PROVIDER'}; my $subscription_code = $cgiparams{'SUBSCRIPTION_CODE'}; @@ -781,8 +783,9 @@ sub show_display_error_message() { ## Function to display the main IDS page. # sub show_mainpage() { - # Read-in idssettings . + # Read-in idssettings and provider settings. &General::readhash("$IDS::ids_settings_file", \%idssettings); + &General::readhasharray("$IDS::providers_settings_file", \%used_providers); # If no autoupdate intervall has been configured yet, set default value. unless(exists($idssettings{'AUTOUPDATE_INTERVAL'})) { @@ -1368,8 +1371,12 @@ END ## Function to show section for add/edit a provider. # sub show_add_provider() { + my %used_providers = (); my @subscription_providers; + # Read -in providers settings file. + &General::readhasharray("$IDS::providers_settings_file", \%used_providers); + # Get all supported ruleset providers. my @ruleset_providers = &IDS::get_ruleset_providers(); From bb4c30c653314754b8564f6bc41047d5c00eacb6 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 17:38:26 +0100 Subject: [PATCH 026/140] ids.cgi: Correctly use "enabled" for checked checkboxes. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 668dc7bcb..f97970cf7 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -675,7 +675,14 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Assign some nice human-readable values. my $provider = $cgiparams{'PROVIDER'}; my $subscription_code = $cgiparams{'SUBSCRIPTION_CODE'}; - my $status_autoupdate = $cgiparams{'ENABLE_AUTOUPDATE'}; + my $status_autoupdate; + + # Handle autoupdate checkbox. + if ($cgiparams{'ENABLE_AUTOUPDATE'} eq "on") { + $status_autoupdate = "enabled"; + } else { + $status_autoupdate = "disabled"; + } # Check if we are going to add a new provider. if ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'add'}") { @@ -1464,7 +1471,7 @@ END # Check if an existing provider should be edited. if($cgiparams{'PROVIDERS'} eq "$Lang::tr{'edit'}") { # Check if autoupdate is enabled for this provider. - if ($used_providers{$cgiparams{'ID'}}[2] eq "on") { + if ($used_providers{$cgiparams{'ID'}}[2] eq "enabled") { # Set the checkbox to be checked. $checked{'ENABLE_AUTOUPDATE'} = "checked='checked'"; } From 2acb3c8d0032cb4056a13d168be9be85f5607df9 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 17:39:32 +0100 Subject: [PATCH 027/140] ids.cgi: Remove accidently commited commented code snipped. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 1 - 1 file changed, 1 deletion(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index f97970cf7..553f583cd 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -986,7 +986,6 @@ END my $col = ""; # Loop through all entries of the hash. - #while( (my $id) = each %used_providers) { foreach my $id (sort keys(%used_providers)) { # Assign data array positions to some nice variable names. my $provider = $used_providers{$id}[0]; From 7323c72d03f063c6847df8973ea6abb09b6b1323 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 17:40:12 +0100 Subject: [PATCH 028/140] ids.cgi: Fix type in method. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 553f583cd..5ceb7e400 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1043,7 +1043,7 @@ print < -
+ From 9bf260ded2713a983c62072ee89772884bc14a8a Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 17:40:44 +0100 Subject: [PATCH 029/140] ids.cgi: Add code to handle enable/disable autoupdate for a provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 5ceb7e400..400a53322 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -666,6 +666,40 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { &reload(); } +# Toggle Enable/Disable autoupdate for a provider +} elsif ($cgiparams{'AUTOUPDATE'} eq $Lang::tr{'toggle enable disable'}) { + my %used_providers = (); + + # Only go further, if an ID has been passed. + if ($cgiparams{'ID'}) { + # Assign the given ID. + my $id = $cgiparams{'ID'}; + + # Undef the given ID. + undef($cgiparams{'ID'}); + + # Read-in providers settings file. + &General::readhasharray($IDS::providers_settings_file, \%used_providers); + + # Grab the configured status of the corresponding entry. + my $status_autoupdate = $used_providers{$id}[2]; + + # Switch the status. + if ($status_autoupdate eq "disabled") { + $status_autoupdate = "enabled"; + } else { + $status_autoupdate = "disabled"; + } + + # Modify the status of the existing entry. + $used_providers{$id} = ["$used_providers{$id}[0]", "$used_providers{$id}[1]", "$status_autoupdate", "$used_providers{$id}[3]"]; + + # Write the changed hash to the providers settings file. + &General::writehasharray($IDS::providers_settings_file, \%used_providers); + } + +# Add/Edit a provider to the list of used providers. +# } elsif (($cgiparams{'PROVIDERS'} eq "$Lang::tr{'add'}") || ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'update'}")) { my %used_providers = (); From 73eb03a333067b680e4c77469eb86da5444bdd3d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Tue, 23 Mar 2021 17:42:07 +0100 Subject: [PATCH 030/140] ids.cgi: Add code to handle enable/disable a provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 75 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 400a53322..b8bdd4fe2 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -789,6 +789,81 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Undefine providers flag. undef($cgiparams{'PROVIDERS'}); } + +## Toggle Enabled/Disabled for an existing provider. +# +} elsif ($cgiparams{'PROVIDERS'} eq $Lang::tr{'toggle enable disable'}) { + my %used_providers = (); + + # Only go further, if an ID has been passed. + if ($cgiparams{'ID'}) { + # Assign the given ID. + my $id = $cgiparams{'ID'}; + + # Undef the given ID. + undef($cgiparams{'ID'}); + + # Read-in file which contains the provider settings. + &General::readhasharray($IDS::providers_settings_file, \%used_providers); + + # Grab the configured status of the corresponding entry. + my $status = $used_providers{$id}[3]; + + # Switch the status. + if ($status eq "enabled") { + $status = "disabled"; + } else { + $status = "enabled"; + } + + # Modify the status of the existing entry. + $used_providers{$id} = ["$used_providers{$id}[0]", "$used_providers{$id}[1]", "$used_providers{$id}[2]", "$status"]; + + # Write the changed hash to the providers settings file. + &General::writehasharray($IDS::providers_settings_file, \%used_providers); + + # XXX - The ruleset needs to be regenerated + # XXX - Suricata requires a reload or if the last provider + # has been disabled suricata needs to be stopped. + # Check if the IDS is running. + #if(&IDS::ids_is_running()) { + # # Call suricatactrl to perform a reload. + # &IDS::call_suricatactrl("reload"); + #} + + # Undefine providers flag. + undef($cgiparams{'PROVIDERS'}); + } + +## Remove provider from the list of used providers. +# +} elsif ($cgiparams{'PROVIDERS'} eq $Lang::tr{'remove'}) { + my %used_providers = (); + + # Read-in provider settings file. + &General::readhasharray($IDS::providers_settings_file, \%used_providers); + + # Drop entry from the hash. + delete($used_providers{$cgiparams{'ID'}}); + + # Undef the given ID. + undef($cgiparams{'ID'}); + + # Write the changed hash to the provide settings file. + &General::writehasharray($IDS::providers_settings_file, \%used_providers); + + # XXX - The ruleset of the provider needs to be dropped. + # XXX - The remain rulest of suricata needs to be regenerated. + # XXX - Suricata requires a reload or if the last provider has + # been removed it has to be stopped. + # Check if the IDS is running. + #if(&IDS::ids_is_running()) { + # Call suricatactrl to perform a reload. + # &IDS::call_suricatactrl("reload"); + #} + + # Undefine providers flag. + undef($cgiparams{'PROVIDERS'}); } &Header::openpage($Lang::tr{'intrusion detection system'}, 1, ''); From 923a644107b608ba6965359f75193fbc34647461 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 26 Mar 2021 13:15:04 +0100 Subject: [PATCH 031/140] ruleset-sources: Replace subscription code placeholder. Replace the placeholder by the more generic . Signed-off-by: Stefan Schantl --- config/suricata/ruleset-sources | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index a4f556118..d82f3391e 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -23,7 +23,7 @@ our %Providers = ( website => "https://www.snort.org", tr_string => "registered user rules", requires_subscription => "True", - dl_url => "https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode=", + dl_url => "https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode=", dl_type => "archive", }, @@ -33,7 +33,7 @@ our %Providers = ( website => "https://www.snort.org", tr_string => "subscripted user rules", requires_subscription => "True", - dl_url => "https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode=", + dl_url => "https://www.snort.org/rules/snortrules-snapshot-29161.tar.gz?oinkcode=", dl_type => "archive", }, @@ -63,7 +63,7 @@ our %Providers = ( website => "https://emergingthreats.net/", tr_string => "emerging pro rules", requires_subscription => "True", - dl_url => "https://rules.emergingthreatspro.com//suricata-5.0/etpro.rules.tar.gz", + dl_url => "https://rules.emergingthreatspro.com//suricata-5.0/etpro.rules.tar.gz", dl_type => "archive", }, ); From b5350c4d6ee39b4991f1fb0d467b87d514b59a58 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 26 Mar 2021 13:16:40 +0100 Subject: [PATCH 032/140] ruleset-sources: Fix website url for community ruleset. Signed-off-by: Stefan Schantl --- config/suricata/ruleset-sources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index d82f3391e..563918e48 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -40,7 +40,7 @@ our %Providers = ( # Community rules from sourcefire. community => { summary => "Snort/VRT GPLv2 Community Rules", - website => "https://www.snort.ort", + website => "https://www.snort.org", tr_string => "community rules", requires_subscription => "False", dl_url => "https://www.snort.org/rules/community", From e55fa2f7456d3f4bf16a33a3f5d06582b9e78de9 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 26 Mar 2021 13:17:59 +0100 Subject: [PATCH 033/140] ids-functions.pl: Run in perl strict mode. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 3e146b589..a71305b98 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -21,6 +21,8 @@ # # ############################################################################ +use strict; + package IDS; require '/var/ipfire/general-functions.pl'; From 788a71f51eedf087b6c91bc5714ed0b1834d202d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 26 Mar 2021 13:19:58 +0100 Subject: [PATCH 034/140] ids-functions.pl: Introduce private _get_dl_rulesfile() function. This function can be used to generate/get the absolute file and path for a given ruleset provider. The files will be stored in the usual "/var/tmp" folder with a new file format based on the dl_file type and the provider. Examples could be: * /var/ipfire/idsrules-emerging.tar.gz * /var/ipfire/idsrules-registered.tar.gz * /var/ipfire/idsrules-somprovider.rules Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 39 ++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index a71305b98..51bc6a3c3 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -65,9 +65,12 @@ our $providers_settings_file = "$settingsdir/providers-settings"; # File which stores the configured settings for whitelisted addresses. our $ignored_file = "$settingsdir/ignored"; -# Location and name of the tarball which contains the ruleset. +# DEPRECATED - Location and name of the tarball which contains the ruleset. our $rulestarball = "/var/tmp/idsrules.tar.gz"; +# Location where the downloaded rulesets are stored. +our $dl_rules_path = "/var/tmp"; + # File to store any errors, which also will be read and displayed by the wui. our $storederrorfile = "/tmp/ids_storederror"; @@ -93,6 +96,9 @@ our $idspidfile = "/var/run/suricata.pid"; # Location of suricatactrl. my $suricatactrl = "/usr/local/bin/suricatactrl"; +# Prefix for each downloaded ruleset. +my $dl_rulesfile_prefix = "idsrules"; + # Array with allowed commands of suricatactrl. my @suricatactrl_cmds = ( 'start', 'stop', 'restart', 'reload', 'fix-rules-dir', 'cron' ); @@ -103,6 +109,12 @@ my @cron_intervals = ('off', 'daily', 'weekly' ); # http_ports_file. my @http_ports = ('80', '81'); +# Hash which allows to convert the download type (dl_type) to a file suffix. +my %dl_type_to_suffix = ( + "archive" => ".tar.gz", + "plain" => ".rules", +); + # ## Function to check and create all IDS related files, if the does not exist. # @@ -431,6 +443,31 @@ sub _store_error_message ($) { &set_ownership("$storederrorfile"); } +# +## Private function to get the path and filename for a downloaded ruleset by a given provider. +# +sub _get_dl_rulesfile($) { + my ($provider) = @_; + + # Gather the download type for the given provider. + my $dl_type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; + + # Obtain the file suffix for the download file type. + my $suffix = $dl_type_to_suffix{$dl_type}; + + # Check if a suffix has been found. + unless ($suffix) { + # Abort return - nothing. + return; + } + + # Generate the full filename and path for the stored rules file. + my $rulesfile = "$dl_rules_path/$dl_rulesfile_prefix-$provider$suffix"; + + # Return the generated filename. + return $rulesfile; +} + # ## Function to check if the IDS is running. # From b3c2c3364dc816a30db72c3ca79370cfded0f345 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 26 Mar 2021 13:24:26 +0100 Subject: [PATCH 035/140] ids-functions.pl: Allow downloadruleset() function to deal with multiple ruleset providers. When calling the function now a single ruleset provider handle can be specified to only download this ruleset or by adding "all" or leaving the handle blank a download of all configured rulesets can be triggered. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 245 +++++++++++++++++++------------- 1 file changed, 149 insertions(+), 96 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 51bc6a3c3..90986094b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -183,23 +183,34 @@ sub checkdiskspace () { } # -## This function is responsible for downloading the configured IDS ruleset. +## This function is responsible for downloading the configured IDS rulesets or if no one is specified +## all configured rulesets will be downloaded. ## -## * At first it obtains from the stored rules settings which ruleset should be downloaded. -## * The next step is to get the download locations for all available rulesets. -## * After that, the function will check if an upstream proxy should be used and grab the settings. -## * The last step will be to generate the final download url, by obtaining the URL for the desired -## ruleset, add the settings for the upstream proxy and final grab the rules tarball from the server. +## * At first it gathers all configured ruleset providers, initialize the downloader and sets an +## upstream proxy if configured. +## * After that, the given ruleset or in case all rulesets should be downloaded, it will determine wether it +## is enabled or not. +## * The next step will be to generate the final download url, by obtaining the URL for the desired +## ruleset, add the settings for the upstream proxy. +## * Finally the function will grab all the rules files or tarballs from the servers. # -sub downloadruleset { - # Get rules settings. - my %rulessettings=(); - &General::readhash("$rules_settings_file", \%rulessettings); +sub downloadruleset ($) { + my ($provider) = @_; + + # If no provider is given default to "all". + $provider //= 'all'; + + # Hash to store the providers and access id's, for which rules should be downloaded. + my %sheduled_providers = (); + + # Get used provider settings. + my %used_providers = (); + &General::readhasharray("$providers_settings_file", \%used_providers); # Check if a ruleset has been configured. - unless($rulessettings{'RULES'}) { + unless(%used_providers) { # Log that no ruleset has been configured and abort. - &_log_to_syslog("No ruleset source has been configured."); + &_log_to_syslog("No ruleset provider has been configured."); # Return "1". return 1; @@ -236,40 +247,100 @@ sub downloadruleset { $downloader->proxy(['http', 'https'], $proxy_url); } - # Grab the right url based on the configured vendor. - my $url = $IDS::Ruleset::Providers{$rulessettings{'RULES'}}{'dl_url'}; + # Loop through the hash of configured providers. + foreach my $id ( keys %used_providers ) { + # Skip providers which are not enabled. + next if ($used_providers{$id}[3] ne "enabled"); - # Check if the vendor requires an oinkcode and add it if needed. - $url =~ s/\/$rulessettings{'OINKCODE'}/g; + # Obtain the provider handle. + my $provider_handle = $used_providers{$id}[0]; - # Abort if no url could be determined for the vendor. - unless ($url) { - # Log error and abort. - &_log_to_syslog("Unable to gather a download URL for the selected ruleset."); - return 1; + # Handle update off all providers. + if (($provider eq "all") || ($provider_handle eq "$provider")) { + # Add provider handle and it's id to the hash of sheduled providers. + $sheduled_providers{$provider_handle} = $id); + } } - # Variable to store the filesize of the remote object. - my $remote_filesize; + # Loop through the hash of sheduled providers. + foreach my $provider ( keys %sheduled_providers) { + # Grab the download url for the provider. + my $url = $IDS::Ruleset::Providers{$provider}{'dl_url'}; - # The sourcfire (snort rules) does not allow to send "HEAD" requests, so skip this check - # for this webserver. - # - # Check if the ruleset source contains "snort.org". - unless ($url =~ /\.snort\.org/) { - # Pass the requrested url to the downloader. - my $request = HTTP::Request->new(HEAD => $url); + # Check if the provider requires a subscription. + if ($IDS::Ruleset::Providers{$provider}{'requires_subscription'} eq "True") { + # Grab the previously stored access id for the provider from hash. + my $id = $sheduled_providers{$provider}; - # Accept the html header. - $request->header('Accept' => 'text/html'); + # Grab the subscription code. + my $subscription_code = $used_providers{$id}[1]; - # Perform the request and fetch the html header. - my $response = $downloader->request($request); + # Add the subscription code to the download url. + $url =~ s/\/$subscription_code/g; + + } + + # Abort if no url could be determined for the provider. + unless ($url) { + # Log error and abort. + &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider."); + return 1; + } + + # Variable to store the filesize of the remote object. + my $remote_filesize; + + # The sourcfire (snort rules) does not allow to send "HEAD" requests, so skip this check + # for this webserver. + # + # Check if the ruleset source contains "snort.org". + unless ($url =~ /\.snort\.org/) { + # Pass the requrested url to the downloader. + my $request = HTTP::Request->new(HEAD => $url); + + # Accept the html header. + $request->header('Accept' => 'text/html'); + + # Perform the request and fetch the html header. + my $response = $downloader->request($request); + + # Check if there was any error. + unless ($response->is_success) { + # Obtain error. + my $error = $response->status_line(); + + # Log error message. + &_log_to_syslog("Unable to download the ruleset. \($error\)"); + + # Return "1" - false. + return 1; + } + + # Assign the fetched header object. + my $header = $response->headers(); + + # Grab the remote file size from the object and store it in the + # variable. + $remote_filesize = $header->content_length; + } + + # Load perl module to deal with temporary files. + use File::Temp; + + # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp". + my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 ); + my $tmpfile = $tmp->filename(); + + # Pass the requested url to the downloader. + my $request = HTTP::Request->new(GET => $url); + + # Perform the request and save the output into the tmpfile. + my $response = $downloader->request($request, $tmpfile); # Check if there was any error. unless ($response->is_success) { # Obtain error. - my $error = $response->status_line(); + my $error = $response->content; # Log error message. &_log_to_syslog("Unable to download the ruleset. \($error\)"); @@ -278,70 +349,52 @@ sub downloadruleset { return 1; } - # Assign the fetched header object. - my $header = $response->headers(); + # Load perl stat module. + use File::stat; - # Grab the remote file size from the object and store it in the - # variable. - $remote_filesize = $header->content_length; + # Perform stat on the tmpfile. + my $stat = stat($tmpfile); + + # Grab the local filesize of the downloaded tarball. + my $local_filesize = $stat->size; + + # Check if both file sizes match. + if (($remote_filesize) && ($remote_filesize ne $local_filesize)) { + # Log error message. + &_log_to_syslog("Unable to completely download the ruleset. "); + &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. "); + + # Delete temporary file. + unlink("$tmpfile"); + + # Return "1" - false. + return 1; + } + + # Genarate and assign file name and path to store the downloaded rules file. + my $dl_rulesfile = &_get_dl_rulesfile($provider); + + # Check if a file name could be obtained. + unless ($dl_rulesfile) { + # Log error message. + &_log_to_syslog("Unable to store the downloaded rules file. "); + + # Delete downloaded temporary file. + unlink("$tmpfile"); + + # Return "1" - false. + } + + # Load file copy module, which contains the move() function. + use File::Copy; + + # Overwrite the may existing rulefile or tarball with the downloaded one. + move("$tmpfile", "$dl_rulesfile"); + + # Set correct ownership for the tarball. + set_ownership("$dl_rulesfile"); } - # Load perl module to deal with temporary files. - use File::Temp; - - # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tar.gz". - my $tmp = File::Temp->new( SUFFIX => ".tar.gz", DIR => "/var/tmp/", UNLINK => 0 ); - my $tmpfile = $tmp->filename(); - - # Pass the requested url to the downloader. - my $request = HTTP::Request->new(GET => $url); - - # Perform the request and save the output into the tmpfile. - my $response = $downloader->request($request, $tmpfile); - - # Check if there was any error. - unless ($response->is_success) { - # Obtain error. - my $error = $response->content; - - # Log error message. - &_log_to_syslog("Unable to download the ruleset. \($error\)"); - - # Return "1" - false. - return 1; - } - - # Load perl stat module. - use File::stat; - - # Perform stat on the tmpfile. - my $stat = stat($tmpfile); - - # Grab the local filesize of the downloaded tarball. - my $local_filesize = $stat->size; - - # Check if both file sizes match. - if (($remote_filesize) && ($remote_filesize ne $local_filesize)) { - # Log error message. - &_log_to_syslog("Unable to completely download the ruleset. "); - &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. "); - - # Delete temporary file. - unlink("$tmpfile"); - - # Return "1" - false. - return 1; - } - - # Load file copy module, which contains the move() function. - use File::Copy; - - # Overwrite existing rules tarball with the new downloaded one. - move("$tmpfile", "$rulestarball"); - - # Set correct ownership for the rulesdir and files. - set_ownership("$rulestarball"); - # If we got here, everything worked fine. Return nothing. return; } From caae0cf5e342070d04fac84c6b85cf1efc3bfe23 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 26 Mar 2021 13:27:48 +0100 Subject: [PATCH 036/140] ruleset-sources: Rename file to plain. This is used if a provider offers a plain rulefile instead an archive. Signed-off-by: Stefan Schantl --- config/suricata/ruleset-sources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/suricata/ruleset-sources b/config/suricata/ruleset-sources index 563918e48..edef12d45 100644 --- a/config/suricata/ruleset-sources +++ b/config/suricata/ruleset-sources @@ -12,7 +12,7 @@ package IDS::Ruleset; # tr_string => The translation string which is used by the WUI and part of the language files. # requires_subscription => "True/False" - If some kind of registration code is required in order to download the ruleset. # dl_url => The download URL to grab the ruleset. -# dl_type => "archive/file" - To specify, if the downloaded file is a packed archive or a plain text file. +# dl_type => "archive/plain" - To specify, if the downloaded file is a packed archive or a plain text file. # }, # Hash which contains the supported ruleset providers. From 2c02c936075c9eac0196530f61729740d8c01142 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 27 Mar 2021 12:06:44 +0100 Subject: [PATCH 037/140] ids-functions.pl: Fix typo. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 90986094b..4f3599d3c 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -258,7 +258,7 @@ sub downloadruleset ($) { # Handle update off all providers. if (($provider eq "all") || ($provider_handle eq "$provider")) { # Add provider handle and it's id to the hash of sheduled providers. - $sheduled_providers{$provider_handle} = $id); + $sheduled_providers{$provider_handle} = $id; } } From ae22613224bcdb93454b3035e2a8f48ee40d147f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 27 Mar 2021 12:07:13 +0100 Subject: [PATCH 038/140] ids-functions.pl: Always delete temporary file. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 4f3599d3c..27f734907 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -391,6 +391,9 @@ sub downloadruleset ($) { # Overwrite the may existing rulefile or tarball with the downloaded one. move("$tmpfile", "$dl_rulesfile"); + # Delete temporary file. + unlink("$tmpfile"); + # Set correct ownership for the tarball. set_ownership("$dl_rulesfile"); } From 0fbfffea9152715705d1c3c9b318635fd81bb89f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 27 Mar 2021 12:07:45 +0100 Subject: [PATCH 039/140] ids-functions.pl: Introduce extraceruleset() function. This function is used to extract the required config and rules files from the stored rules tarball for a given ruleset provider. * The files will be extracted to a temporary directory layout in "/tmp/ids_tmp". * Names of config files will be adjusted in case multiple providers offers the same config files, which is very common. * The name of the single rulefiles will be adjusted to start with the vendors name to allow assigning them very easily to a single ruleset provider. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 101 ++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 27f734907..8ee56a07e 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -99,6 +99,9 @@ my $suricatactrl = "/usr/local/bin/suricatactrl"; # Prefix for each downloaded ruleset. my $dl_rulesfile_prefix = "idsrules"; +# Temporary directory where the rulesets will be extracted. +my $tmp_directory = "/tmp/ids_tmp"; + # Array with allowed commands of suricatactrl. my @suricatactrl_cmds = ( 'start', 'stop', 'restart', 'reload', 'fix-rules-dir', 'cron' ); @@ -402,6 +405,104 @@ sub downloadruleset ($) { return; } +# +## Function to extract a given ruleset. +# +sub extractruleset ($) { + my ($provider) = @_; + + # Load perl module to deal with archives. + use Archive::Tar; + + # Load perl module to deal with files and path. + use File::Basename; + + # Get full path and downloaded rulesfile for the given provider. + my $tarball = &_get_dl_rulesfile($provider); + + # Check if the file exists. + unless (-f $tarball) { + &_log_to_syslog("Could not extract ruleset file: $tarball"); + + # Return nothing. + return; + } + + # Destination directories, where the files will be extracted. + my $rules_destdir = "$tmp_directory/rules"; + my $conf_destdir = "$tmp_directory/conf"; + + # Check if the temporary directories exist, otherwise create them. + mkdir("$tmp_directory") unless (-d "$tmp_directory"); + mkdir("$rules_destdir") unless (-d "$rules_destdir"); + mkdir("$conf_destdir") unless (-d "$conf_destdir"); + + # Initialize the tar module. + my $tar = Archive::Tar->new($tarball); + + # Get the filelist inside the tarball. + my @packed_files = $tar->list_files; + + # Loop through the filelist. + foreach my $packed_file (@packed_files) { + my $destination; + + # Splitt the packed file into chunks. + my $file = fileparse($packed_file); + + # Handle msg-id.map file. + if ("$file" eq "sid-msg.map") { + # Set extract destination to temporary config_dir. + $destination = "$conf_destdir/$provider\-sid-msg.map"; + # Handle classification.conf + } elsif ("$file" eq "classification.config") { + # Set extract destination to temporary config_dir. + $destination = "$conf_destdir/$provider\-classification.config"; + # Handle rules files. + } elsif ($file =~ m/\.rules$/) { + my $rulesfilename; + + # Splitt the filename into chunks. + my @filename = split("-", $file); + + # Reverse the array. + @filename = reverse(@filename); + + # Get the amount of elements in the array. + my $elements = @filename; + + # Remove last element of the hash. + # It contains the vendor name, which will be replaced. + if ($elements >= 3) { + # Remove last element from hash. + pop(@filename); + } + + # Check if the last element of the filename does not + # contain the providers name. + if (@filename[-1] ne "$provider") { + # Add provider name as last element. + push(@filename, $provider); + } + + # Reverse the array back. + @filename = reverse(@filename); + + # Generate the name for the rulesfile. + $rulesfilename = join("-", @filename); + + # Set extract destination to temporaray rules_dir. + $destination = "$rules_destdir/$rulesfilename"; + } else { + # Skip all other files. + next; + } + + # Extract the file to the temporary directory. + $tar->extract_file("$packed_file", "$destination"); + } +} + # ## A tiny wrapper function to call the oinkmaster script. # From 23b560529ad02c3d6eac37ed60eacf5af99be69d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 28 Mar 2021 12:47:23 +0200 Subject: [PATCH 040/140] ids-functions.pl: Introduce merge_classifications() function. This function is used to merge the individual classification files provided by the providers. The result will be written to the classification.config which will be used by the IDS. Fixes #11884. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 8ee56a07e..f9bf07af3 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -80,6 +80,9 @@ our $ids_page_lock_file = "/tmp/ids_page_locked"; # Location where the rulefiles are stored. our $rulespath = "/var/lib/suricata"; +# Location of the classification file. +our $classification_file = "$rulespath/classification.config"; + # Location to store local rules. This file will not be touched. our $local_rules_file = "$rulespath/local.rules"; @@ -539,6 +542,71 @@ sub oinkmaster () { closelog(); } +# +## Function to merge the classifications for a given amount of providers and write them +## to the classifications file. +# +sub merge_classifications(@) { + my @providers = @_; + + # Hash to store all collected classifications. + my %classifications = (); + + # Loop through the given array of providers. + foreach my $provider (@providers) { + # Generate full path to classification file. + my $classification_file = "$tmp_directory/conf/$provider\-classification.config"; + + # Skip provider if no classification file exists. + next unless (-f "$classification_file"); + + # Open the classification file. + open(CLASSIFICATION, $classification_file) or die "Could not open file $classification_file. $!\n"; + + # Loop through the file content. + while() { + # Parse the file and grab the classification details. + if ($_ =~/.*config classification\: (.*)/) { + # Split the grabbed details. + my ($short_name, $short_desc, $priority) = split("\,", $1); + + # Check if the grabbed classification is allready known and the priority value is greater + # than the stored one (which causes less priority in the IDS). + if (($classifications{$short_name}) && ($classifications{$short_name}[1] >= $priority)) { + #Change the priority value to the stricter one. + $classifications{$short_name} = [ "$classifications{$short_name}[0]", "$priority" ]; + } else { + # Add the classification to the hash. + $classifications{$short_name} = [ "$short_desc", "$priority" ]; + } + } + } + + # Close the file. + close(CLASSIFICATION); + } + + # Open classification file for writing. + open(FILE, ">", "$classification_file") or die "Could not write to $classification_file. $!\n"; + + # Print notice about autogenerated file. + print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n"; + + # Sort and loop through the hash of classifications. + foreach my $key (sort keys %classifications) { + # Assign some nice variable names for the items. + my $short_name = $key; + my $short_desc = $classifications{$key}[0]; + my $priority = $classifications{$key}[1]; + + # Write the classification to the file. + print FILE "config classification: $short_name,$short_desc,$priority\n"; + } + + # Close file handle. + close(FILE); +} + # ## Function to do all the logging stuff if the downloading or updating of the ruleset fails. # From 8335286b381381e187da787c0c758fa62890104e Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 28 Mar 2021 12:49:56 +0200 Subject: [PATCH 041/140] ids-functions.pl: Fix typo. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index f9bf07af3..359d7d4bd 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -483,7 +483,7 @@ sub extractruleset ($) { # Check if the last element of the filename does not # contain the providers name. - if (@filename[-1] ne "$provider") { + if ($filename[-1] ne "$provider") { # Add provider name as last element. push(@filename, $provider); } From 8bd74e12a9433f0f79e9eeca1028192799c98cc7 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 29 Mar 2021 15:27:42 +0200 Subject: [PATCH 042/140] ids-functions.pl: Introduce merge_sid_msg() function. This function is used to merge the sid to message mapping files from various providers. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 72 +++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 359d7d4bd..cd15b350d 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -83,6 +83,9 @@ our $rulespath = "/var/lib/suricata"; # Location of the classification file. our $classification_file = "$rulespath/classification.config"; +# Location of the sid to msg mappings file. +our $sid_msg_file = "$rulespath/sid-msg.map"; + # Location to store local rules. This file will not be touched. our $local_rules_file = "$rulespath/local.rules"; @@ -607,6 +610,75 @@ sub merge_classifications(@) { close(FILE); } +# +## Function to merge the "sid to message mapping" files of various given providers. +# +sub merge_sid_msg (@) { + my @providers = @_; + + # Hash which contains all the sid to message mappings. + my %mappings = (); + + # Loop through the array of given providers. + foreach my $provider (@providers) { + # Generate full path and filename. + my $sid_msg_file = "$tmp_directory/conf/$provider\-sid-msg.map"; + + # Skip provider if no sid to msg mapping file for this provider exists. + next unless (-f $sid_msg_file); + + # Open the file. + open(MAPPING, $sid_msg_file) or die "Could not open $sid_msg_file. $!\n"; + + # Loop through the file content. + while () { + # Remove newlines. + chomp($_); + + # Skip lines which do not start with a number, + next unless ($_ =~ /^\d+/); + + # Split line content and assign it to an array. + my @line = split(/ \|\| /, $_); + + # Grab the first element (and remove it) from the line array. + # It contains the sid. + my $sid = shift(@line); + + # Store the grabbed sid and the remain array as hash value. + # It still contains the messages, references etc. + $mappings{$sid} = [@line]; + } + + # Close file handle. + close(MAPPING); + } + + # Open mappings file for writing. + open(FILE, ">", $sid_msg_file) or die "Could not write $sid_msg_file. $!\n"; + + # Write notice about autogenerated file. + print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n"; + + # Loop through the hash of mappings. + foreach my $sid ( sort keys %mappings) { + # Grab data for the sid. + my @data = @{$mappings{$sid}}; + + # Add the sid to the data array. + unshift(@data, $sid); + + # Generate line. + my $line = join(" \|\| ", @data); + + print FILE "$line\n"; + + } + + # Close file handle. + close(FILE); +} + # ## Function to do all the logging stuff if the downloading or updating of the ruleset fails. # From 6c9d3eeef24be039b13c12e2ed750556f79a2b04 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 29 Mar 2021 15:50:04 +0200 Subject: [PATCH 043/140] ids-functions.pl: Assign temporary rules and conf path to variables. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index cd15b350d..e5c191043 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -108,6 +108,12 @@ my $dl_rulesfile_prefix = "idsrules"; # Temporary directory where the rulesets will be extracted. my $tmp_directory = "/tmp/ids_tmp"; +# Temporary directory where the extracted rules files will be stored. +my $tmp_rules_directory = "$tmp_directory/rules"; + +# Temporary directory where the extracted additional config files will be stored. +my $tmp_conf_directory = "$tmp_directory/conf"; + # Array with allowed commands of suricatactrl. my @suricatactrl_cmds = ( 'start', 'stop', 'restart', 'reload', 'fix-rules-dir', 'cron' ); @@ -434,14 +440,10 @@ sub extractruleset ($) { return; } - # Destination directories, where the files will be extracted. - my $rules_destdir = "$tmp_directory/rules"; - my $conf_destdir = "$tmp_directory/conf"; - # Check if the temporary directories exist, otherwise create them. mkdir("$tmp_directory") unless (-d "$tmp_directory"); - mkdir("$rules_destdir") unless (-d "$rules_destdir"); - mkdir("$conf_destdir") unless (-d "$conf_destdir"); + mkdir("$tmp_rules_directory") unless (-d "$tmp_rules_directory"); + mkdir("$tmp_conf_directory") unless (-d "$tmp_conf_directory"); # Initialize the tar module. my $tar = Archive::Tar->new($tarball); @@ -459,11 +461,11 @@ sub extractruleset ($) { # Handle msg-id.map file. if ("$file" eq "sid-msg.map") { # Set extract destination to temporary config_dir. - $destination = "$conf_destdir/$provider\-sid-msg.map"; + $destination = "$tmp_conf_directory/$provider\-sid-msg.map"; # Handle classification.conf } elsif ("$file" eq "classification.config") { # Set extract destination to temporary config_dir. - $destination = "$conf_destdir/$provider\-classification.config"; + $destination = "$tmp_conf_directory/$provider\-classification.config"; # Handle rules files. } elsif ($file =~ m/\.rules$/) { my $rulesfilename; @@ -498,7 +500,7 @@ sub extractruleset ($) { $rulesfilename = join("-", @filename); # Set extract destination to temporaray rules_dir. - $destination = "$rules_destdir/$rulesfilename"; + $destination = "$tmp_rules_directory/$rulesfilename"; } else { # Skip all other files. next; @@ -558,7 +560,7 @@ sub merge_classifications(@) { # Loop through the given array of providers. foreach my $provider (@providers) { # Generate full path to classification file. - my $classification_file = "$tmp_directory/conf/$provider\-classification.config"; + my $classification_file = "$tmp_conf_directory/$provider\-classification.config"; # Skip provider if no classification file exists. next unless (-f "$classification_file"); @@ -622,7 +624,7 @@ sub merge_sid_msg (@) { # Loop through the array of given providers. foreach my $provider (@providers) { # Generate full path and filename. - my $sid_msg_file = "$tmp_directory/conf/$provider\-sid-msg.map"; + my $sid_msg_file = "$tmp_conf_directory/$provider\-sid-msg.map"; # Skip provider if no sid to msg mapping file for this provider exists. next unless (-f $sid_msg_file); From b953677b0d05202f69bb2ef06e9b628c39ea37f2 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 29 Mar 2021 16:51:18 +0200 Subject: [PATCH 044/140] ids-functions.pl: Rework oinkmaster() function. Rework the function to work with the latest changes and multiple providers. The function now does the following: * Extract the stored rules tarballs for all enabled providers. * Copy rules files for enabled providers which provide plain files. * Still calls oinkmaster to set up the rules and modify them. * Calls the merge functions for classification and sid to msg files. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 66 +++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index e5c191043..daa004493 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -512,15 +512,61 @@ sub extractruleset ($) { } # -## A tiny wrapper function to call the oinkmaster script. +## A wrapper function to call the oinkmaster script, setup the rules structues and +## call the functions to merge the additional config files. (classification, sid-msg, etc.). # sub oinkmaster () { + # Load perl module for file copying. + use File::Copy; + + # Hash to store the used providers. + my %used_providers = (); + + # Array to store the enabled providers. + my @enabled_providers = (); + # Check if the files in rulesdir have the correct permissions. &_check_rulesdir_permissions(); - # Cleanup the rules directory before filling it with the new rulest. + # Cleanup the rules directory before filling it with the new rulests. &_cleanup_rulesdir(); + # Read-in the providers config file. + &General::readhasharray("$providers_settings_file", \%used_providers); + + # Loop through the hash of used_providers. + foreach my $id (keys %used_providers) { + # Skip disabled providers. + next unless ($used_providers{$id}[3] eq "enabled"); + + # Grab the provider handle. + my $provider = "$used_providers{$id}[0]"; + + # Add the provider handle to the array of enabled providers. + push(@enabled_providers, $provider); + + # Omit the type (dl_type) of the stored ruleset. + my $type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; + + # Handle the different ruleset types. + if ($type eq "archive") { + # Call the extractruleset function. + &extractruleset($provider); + } elsif ($type eq "plain") { + # Generate filename and full path for the stored rulesfile. + my $dl_rulesfile = &_get_dl_rulesfile($provider); + + # Generate destination filename an full path. + my $destination = "$tmp_rules_directory/$provider\-ruleset.rules"; + + # Copy the file into the temporary rules directory. + copy($dl_rulesfile, $destination); + } else { + # Skip unknown type. + next; + } + } + # Load perl module to talk to the kernel syslog. use Sys::Syslog qw(:DEFAULT setlogsock); @@ -528,7 +574,7 @@ sub oinkmaster () { openlog('oinkmaster', 'cons,pid', 'user'); # Call oinkmaster to generate ruleset. - open(OINKMASTER, "/usr/local/bin/oinkmaster.pl -s -u file://$rulestarball -C $settingsdir/oinkmaster.conf -o $rulespath 2>&1 |") or die "Could not execute oinkmaster $!\n"; + open(OINKMASTER, "/usr/local/bin/oinkmaster.pl -s -u dir://$tmp_rules_directory -C $settingsdir/oinkmaster.conf -o $rulespath 2>&1 |") or die "Could not execute oinkmaster $!\n"; # Log output of oinkmaster to syslog. while() { @@ -545,6 +591,20 @@ sub oinkmaster () { # Close the log handle. closelog(); + + use Data::Dumper; + + print Dumper \@enabled_providers; + + # Call function to merge the classification files. + &merge_classifications(@enabled_providers); + + # Call function to merge the sid to message mapping files. + &merge_sid_msg(@enabled_providers); + + # Cleanup temporary directory. + # XXX - not implemented yet. + # &cleanup_tmp_directory(); } # From 09f7de97732b12c7bc13b7f7a9b664a975416647 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 29 Mar 2021 16:53:52 +0200 Subject: [PATCH 045/140] ids-functions.pl: Remove config files on rulesdir cleanup. They every time oinkmaster is called will be generated. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index daa004493..1602c42a9 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -966,9 +966,6 @@ sub _cleanup_rulesdir() { # We only want files. next unless (-f "$rulespath/$file"); - # Skip element if it has config as file extension. - next if ($file =~ m/\.config$/); - # Skip rules file for whitelisted hosts. next if ("$rulespath/$file" eq $whitelist_file); From 16b2d281ce054a41cbe084d7770fc54553ed747d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 10:45:14 +0200 Subject: [PATCH 046/140] ids-functions.pl: Add cleanup_tmp_directory() function. As the name of the function already says, it is responsible to delete all temporary files after ruleset generation. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 1602c42a9..7d2ae802e 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -603,8 +603,7 @@ sub oinkmaster () { &merge_sid_msg(@enabled_providers); # Cleanup temporary directory. - # XXX - not implemented yet. - # &cleanup_tmp_directory(); + &cleanup_tmp_directory(); } # @@ -741,6 +740,17 @@ sub merge_sid_msg (@) { close(FILE); } +# +## Function to cleanup the temporary IDS directroy. +# +sub cleanup_tmp_directory () { + # Load rmtree() function from file path perl module. + use File::Path 'rmtree'; + + # Delete temporary directory and all containing files. + rmtree([ "$tmp_directory" ]); +} + # ## Function to do all the logging stuff if the downloading or updating of the ruleset fails. # From aac869c47ef06294da337e80a1794b0b389e33f4 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 10:49:19 +0200 Subject: [PATCH 047/140] ids-functions.pl: Rework function for modify-sid file to be more generic. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 7d2ae802e..89fee83ff 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1214,12 +1214,7 @@ sub write_used_rulefiles_file(@) { sub write_modify_sids_file() { # Get configured settings. my %idssettings=(); - my %rulessettings=(); &General::readhash("$ids_settings_file", \%idssettings); - &General::readhash("$rules_settings_file", \%rulessettings); - - # Gather the configured ruleset. - my $ruleset = $rulessettings{'RULES'}; # Open modify sid's file for writing. open(FILE, ">$modify_sids_file") or die "Could not write to $modify_sids_file. $!\n"; @@ -1236,33 +1231,24 @@ sub write_modify_sids_file() { # malware in that file. Rules which fall into the first category should stay as # alert since not all flows of that type contain malware. - if($ruleset eq 'registered' or $ruleset eq 'subscripted' or $ruleset eq 'community') { - # These types of rulesfiles contain meta-data which gives the action that should - # be used when in IPS mode. Do the following: - # - # 1. Disable all rules and set the action to 'drop' - # 2. Set the action back to 'alert' if the rule contains 'flowbits:noalert;' - # This should give rules not in the policy a reasonable default if the user - # manually enables them. - # 3. Enable rules and set actions according to the meta-data strings. + # These types of rulesfiles contain meta-data which gives the action that should + # be used when in IPS mode. Do the following: + # + # 1. Disable all rules and set the action to 'drop' + # 2. Set the action back to 'alert' if the rule contains 'flowbits:noalert;' + # This should give rules not in the policy a reasonable default if the user + # manually enables them. + # 3. Enable rules and set actions according to the meta-data strings. - my $policy = 'balanced'; # Placeholder to allow policy to be changed. + my $policy = 'balanced'; # Placeholder to allow policy to be changed. print FILE < Date: Wed, 31 Mar 2021 11:26:24 +0200 Subject: [PATCH 048/140] ids-functions.pl: Introduce drop_dl_rulesfile(). This tiny function is used, to delete the stored rulesfile in case a provider will be deleted. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 89fee83ff..ef4c74a57 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -837,6 +837,22 @@ sub _get_dl_rulesfile($) { return $rulesfile; } +# +## Tiny function to delete the stored ruleset file or tarball for a given provider. +# +sub drop_dl_rulesfile ($) { + my ($provider) = @_; + + # Gather the full path and name of the stored rulesfile. + my $rulesfile = &_get_dl_rulesfile($provider); + + # Check if the given rulesfile exists. + if (-f $rulesfile) { + # Delete the stored rulesfile. + unlink($rulesfile) or die "Could not delete $rulesfile. $!\n"; + } +} + # ## Function to check if the IDS is running. # From 6563d44997434a442d6162571e8594dc8796c973 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:01:22 +0200 Subject: [PATCH 049/140] ids-functions.pl: Introduce get_used_rulesfiles() function. This function simply returns an array which contains the used rulesfiles files. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index ef4c74a57..519488d6a 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1476,6 +1476,48 @@ sub get_red_address() { return; } +# +## Function to get all used rulesfiles files. +# +sub get_used_rulesfiles() { + # Array to store the used rulefiles. + my @used_rulesfiles = (); + + # Check if the used rulesfile is empty. + unless (-z $used_rulesfiles_file) { + # Open the file or used rulefiles and read-in content. + open(FILE, $used_rulefiles_file) or die "Could not open $used_rulefiles_file. $!\n"; + + while () { + # Assign the current line to a nice variable. + my $line = $_; + + # Remove newlines. + chomp($line); + + # Skip comments. + next if ($line =~ /\#/); + + # Skip blank lines. + next if ($line =~ /^\s*$/); + + # Gather the rulefile. + if ($line =~ /.*- (.*)/) { + my $rulefile = $1; + + # Add the rulefile to the array of used rulesfiles. + push(@used_rulesfiles, $rulefile); + } + } + + # Close the file. + close(FILE); + } + + # Return the array of used rulesfiles. + return @used_rulesfiles; +} + # ## Function to write the lock file for locking the WUI, while ## the autoupdate script runs. From 3daa30002576490ec04a290dc777d3060c51d4a0 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:02:27 +0200 Subject: [PATCH 050/140] ids.cgi: Use get_used_rulesfiles function from ids-functions.pl. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 43 +++++++++---------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index b8bdd4fe2..0a825425e 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -282,41 +282,16 @@ if ($cgiparams{'RULESET'}) { closedir(DIR); # Gather used rulefiles. - # - # Check if the file for activated rulefiles is not empty. - if(-f $IDS::used_rulefiles_file) { - # Open the file for used rulefile and read-in content. - open(FILE, $IDS::used_rulefiles_file) or die "Could not open $IDS::used_rulefiles_file. $!\n"; + my @used_rulesfiles = &IDS::get_used_rulesfiles(); - # Read-in content. - my @lines = ; - - # Close file. - close(FILE); - - # Loop through the array. - foreach my $line (@lines) { - # Remove newlines. - chomp($line); - - # Skip comments. - next if ($line =~ /\#/); - - # Skip blank lines. - next if ($line =~ /^\s*$/); - - # Gather rule sid and message from the ruleline. - if ($line =~ /.*- (.*)/) { - my $rulefile = $1; - - # Check if the current rulefile exists in the %idsrules hash. - # If not, the file probably does not exist anymore or contains - # no rules. - if($idsrules{$rulefile}) { - # Add the rulefile state to the %idsrules hash. - $idsrules{$rulefile}{'Rulefile'}{'State'} = "on"; - } - } + # Loop through the array of used rulesfiles. + foreach my $rulesfile (@used_rulesfiles) { + # Check if the current rulefile exists in the %idsrules hash. + # If not, the file probably does not exist anymore or contains + # no rules. + if($idsrules{$rulefile}) { + # Add the rulefile state to the %idsrules hash. + $idsrules{$rulefile}{'Rulefile'}{'State'} = "on"; } } } From dae33250b2557e2650f1ac6fb8ced88d33c76ec7 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:16:01 +0200 Subject: [PATCH 051/140] ids-functions.pl: Fix typo. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 519488d6a..1b074050b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1484,7 +1484,7 @@ sub get_used_rulesfiles() { my @used_rulesfiles = (); # Check if the used rulesfile is empty. - unless (-z $used_rulesfiles_file) { + unless (-z $used_rulefiles_file) { # Open the file or used rulefiles and read-in content. open(FILE, $used_rulefiles_file) or die "Could not open $used_rulefiles_file. $!\n"; From 5e20d6cb28a87fca71abd9d4e0b811f6674fd39a Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:16:24 +0200 Subject: [PATCH 052/140] ids-functions.pl: Introduce get_enabled_providers() function. This function simply returns an array with all enabled ruleset providers. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 1b074050b..9424ce567 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -163,6 +163,36 @@ sub get_ruleset_providers() { return sort(@providers); } +# +## Function to get a list of all enabled ruleset providers. +## +## They will be returned as an array. +# +sub get_enabled_providers () { + my %used_providers = (); + + # Array to store the enabled providers. + my @enabled_providers = (); + + # Read-in the providers config file. + &General::readhasharray("$providers_settings_file", \%used_providers); + + # Loop through the hash of used_providers. + foreach my $id (keys %used_providers) { + # Skip disabled providers. + next unless ($used_providers{$id}[3] eq "enabled"); + + # Grab the provider handle. + my $provider = "$used_providers{$id}[0]"; + + # Add the provider to the array of enabled providers. + push(@enabled_providers, $provider); + } + + # Return the array. + return @enabled_providers; +} + # ## Function for checking if at least 300MB of free disk space are available ## on the "/var" partition. From 0130e0d1e1168581ac3bc90d8773d968b1b5c4eb Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:21:41 +0200 Subject: [PATCH 053/140] ids-functions.pl: Rework oinkmaster() to use get_enabled_providers function. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 9424ce567..98df831d9 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -549,32 +549,17 @@ sub oinkmaster () { # Load perl module for file copying. use File::Copy; - # Hash to store the used providers. - my %used_providers = (); - - # Array to store the enabled providers. - my @enabled_providers = (); - # Check if the files in rulesdir have the correct permissions. &_check_rulesdir_permissions(); # Cleanup the rules directory before filling it with the new rulests. &_cleanup_rulesdir(); - # Read-in the providers config file. - &General::readhasharray("$providers_settings_file", \%used_providers); - - # Loop through the hash of used_providers. - foreach my $id (keys %used_providers) { - # Skip disabled providers. - next unless ($used_providers{$id}[3] eq "enabled"); - - # Grab the provider handle. - my $provider = "$used_providers{$id}[0]"; - - # Add the provider handle to the array of enabled providers. - push(@enabled_providers, $provider); + # Get all enabled providers. + my @enabled_provides = &get_enabled_providers(); + # Loop through the array of enabled providers. + foreach my $provider (@enabled_providers) { # Omit the type (dl_type) of the stored ruleset. my $type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; From 6acaa5fa6f6fb4546f058eebc774914e5706ceb3 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:22:17 +0200 Subject: [PATCH 054/140] ids-functions.pl: Remove accidently commited debug code. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 98df831d9..7988728fa 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -607,10 +607,6 @@ sub oinkmaster () { # Close the log handle. closelog(); - use Data::Dumper; - - print Dumper \@enabled_providers; - # Call function to merge the classification files. &merge_classifications(@enabled_providers); From e31458de4eea69d01a81e24bba85b3b655f7ae1f Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 12:31:18 +0200 Subject: [PATCH 055/140] ids-functions.pl: Fix another typo. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 7988728fa..4674d6f88 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -556,7 +556,7 @@ sub oinkmaster () { &_cleanup_rulesdir(); # Get all enabled providers. - my @enabled_provides = &get_enabled_providers(); + my @enabled_providers = &get_enabled_providers(); # Loop through the array of enabled providers. foreach my $provider (@enabled_providers) { From 50f348f681102eae5dc6d26f19292389397e77fb Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 13:39:43 +0200 Subject: [PATCH 056/140] ids-functions.pl: Introduce move_tmp_ruleset() function. This function is used to move an extracted temporary ruleset to the rules location. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 4674d6f88..ed9bb203b 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -751,6 +751,27 @@ sub merge_sid_msg (@) { close(FILE); } +# +## A very tiny function to move an extracted ruleset from the temporary directory into +## the rules directory. +# +sub move_tmp_ruleset() { + # Load perl module. + use File::Copy; + + # Do a directory listing of the temporary directory. + opendir DH, $tmp_rules_directory; + + # Loop over all files. + while(my $file = readdir DH) { + # Move them to the rules directory. + move "$tmp_rules_directory/$file" , "$rulespath/$file"; + } + + # Close directory handle. + closedir DH; +} + # ## Function to cleanup the temporary IDS directroy. # From b734df0e1299407e59d38d9054065305f9c9eb00 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Wed, 31 Mar 2021 13:41:28 +0200 Subject: [PATCH 057/140] ids.cgi: Add action if a new provider is added. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 0a825425e..a1be76b79 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -761,6 +761,28 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Write the changed hash to the providers settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); + # Check if a new provider will be added. + if ($cgiparams{'PROVIDERS'} eq $Lang::tr{'add'}) { + # Lock the webpage and print notice about downloading + # a new ruleset. + &working_notice("$Lang::tr{'ids working'}"); + + # Download the ruleset. + &IDS::downloadruleset($provider); + + # Extract the ruleset + &IDS::extractruleset($provider); + + # Move the ruleset. + &IDS::move_tmp_ruleset(); + + # Cleanup temporary directory. + &IDS::cleanup_tmp_directory(); + + # Perform a reload of the page. + &reload(); + } + # Undefine providers flag. undef($cgiparams{'PROVIDERS'}); } From ddaf8ae1a87902389288904280c055e4601dc4ba Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 11:39:57 +0200 Subject: [PATCH 058/140] IDS: Redesign backend for used provider rulesfiles. The selected rulesfiles of a provider now will be written to an own provider exclusive yaml file, which will be included dynamically when the provider is enabled or not. This allows very easy handling to enable or disable a provider, in this case the file which keeps the enabled providers rulesets only needs to be included in the main file or even not. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 85 +++++++++++++++++++++++++++------ config/suricata/suricata.yaml | 4 +- html/cgi-bin/ids.cgi | 74 +++++++++++++++++++++++----- 3 files changed, 136 insertions(+), 27 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index ed9bb203b..245a267bf 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -27,12 +27,15 @@ package IDS; require '/var/ipfire/general-functions.pl'; require "${General::swroot}/network-functions.pl"; -require "${General::swroot}/suricata/ruleset-sources"; +require "${General::swroot}/suricata/ruleset-sources-new"; # Location where all config and settings files are stored. our $settingsdir = "${General::swroot}/suricata"; -# File where the used rulefiles are stored. +# File where the main file for providers ruleset inclusion exists. +our $suricata_used_providers_file = "$settingsdir/suricata-used-providers.yaml"; + +# DEPRECATED - File where the used rulefiles are stored. our $used_rulefiles_file = "$settingsdir/suricata-used-rulefiles.yaml"; # File where the addresses of the homenet are stored. @@ -138,7 +141,7 @@ sub check_and_create_filelayout() { unless (-f "$enabled_sids_file") { &create_empty_file($enabled_sids_file); } unless (-f "$disabled_sids_file") { &create_empty_file($disabled_sids_file); } unless (-f "$modify_sids_file") { &create_empty_file($modify_sids_file); } - unless (-f "$used_rulefiles_file") { &create_empty_file($used_rulefiles_file); } + unless (-f "$suricata_used_providers_file") { &create_empty_file($suricata_used_providers_file); } unless (-f "$ids_settings_file") { &create_empty_file($ids_settings_file); } unless (-f "$providers_settings_file") { &create_empty_file($providers_settings_file); } unless (-f "$ignored_file") { &create_empty_file($ignored_file); } @@ -1226,13 +1229,18 @@ sub generate_http_ports_file() { } # -## Function to generate and write the file for used rulefiles. +## Function to generate and write the file for used rulefiles file for a given provider. +## +## The function requires as first argument a provider handle, and as second an array with files. # -sub write_used_rulefiles_file(@) { - my @files = @_; +sub write_used_provider_rulefiles_file($@) { + my ($provider, @files) = @_; + + # Get the path and file for the provider specific used rulefiles file. + my $used_provider_rulesfile_file = &get_used_provider_rulesfile_file($provider); # Open file for used rulefiles. - open (FILE, ">$used_rulefiles_file") or die "Could not write to $used_rulefiles_file. $!\n"; + open (FILE, ">$used_provider_rulesfile_file") or die "Could not write to $used_provider_rulesfile_file. $!\n"; # Write yaml header to the file. print FILE "%YAML 1.1\n"; @@ -1241,9 +1249,6 @@ sub write_used_rulefiles_file(@) { # Write header to file. print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - # Allways use the whitelist. - print FILE " - whitelist.rules\n"; - # Loop through the array of given files. foreach my $file (@files) { # Check if the given filename exists and write it to the file of used rulefiles. @@ -1256,6 +1261,53 @@ sub write_used_rulefiles_file(@) { close(FILE); } +# +## Function to write the main file for provider rulesfiles inclusions. +## +## This function requires an array of provider handles. +# +sub write_main_used_rulefiles_file (@) { + my (@providers) = @_; + + # Open file for used rulefils inclusion. + open (FILE, ">", "$suricata_used_providers_file") or die "Could not write to $suricata_used_providers_file. $!\n"; + + # Write yaml header to the file. + print FILE "%YAML 1.1\n"; + print FILE "---\n\n"; + + # Write header to file. + print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; + + # Loop through the list of given providers. + foreach my $provider (@providers) { + # Call function to get the providers used rulefiles file. + my $filename = &get_used_provider_rulesfile_file($provider); + + # Print the provider to the file. + print FILE "include\: $filename\n"; + } + + # XXX - whitelist.rules is not allowed directly, needs to be in a yaml file which has to be included. + # Always use the whitelist file. + #print FILE "\n - whitelist.rules\n"; + + # Close the filehandle after writing. + close(FILE); +} + +# +## Tiny function to generate the full path and name for the used_provider_rulesfile file of a given provider. +# +sub get_used_provider_rulesfile_file ($) { + my ($provider) = @_; + + my $filename = "$settingsdir/suricata\-$provider\-used\-rulefiles.yaml"; + + # Return the gernerated file. + return $filename; +} + # ## Function to generate and write the file for modify the ruleset. # @@ -1509,16 +1561,21 @@ sub get_red_address() { } # -## Function to get all used rulesfiles files. +## Function to get the used rules files of a given provider. # -sub get_used_rulesfiles() { +sub read_used_provider_rulesfiles($) { + my ($provider) = @_; + # Array to store the used rulefiles. my @used_rulesfiles = (); + # Get the used rulesefile file for the provider. + my $rulesfile_file = &get_used_provider_rulesfile_file($provider); + # Check if the used rulesfile is empty. - unless (-z $used_rulefiles_file) { + unless (-z $rulesfile_file) { # Open the file or used rulefiles and read-in content. - open(FILE, $used_rulefiles_file) or die "Could not open $used_rulefiles_file. $!\n"; + open(FILE, $rulesfile_file) or die "Could not open $rulesfile_file. $!\n"; while () { # Assign the current line to a nice variable. diff --git a/config/suricata/suricata.yaml b/config/suricata/suricata.yaml index b4a188d40..84f50f54e 100644 --- a/config/suricata/suricata.yaml +++ b/config/suricata/suricata.yaml @@ -46,8 +46,8 @@ vars: ## default-rule-path: /var/lib/suricata rule-files: - # Include enabled ruleset files from external file - include: /var/ipfire/suricata/suricata-used-rulefiles.yaml + # Include enabled ruleset files from external file. + include: /var/ipfire/suricata/suricata-used-providers.yaml # Include default rules. include: /var/ipfire/suricata/suricata-default-rules.yaml diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index a1be76b79..908ad9c28 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -254,6 +254,10 @@ if (-e $IDS::storederrorfile) { if ($cgiparams{'RULESET'}) { ## Grab all available rules and store them in the idsrules hash. # + + # Get enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + # Open rules directory and do a directory listing. opendir(DIR, $IDS::rulespath) or die $!; # Loop through the direcory. @@ -274,6 +278,15 @@ if ($cgiparams{'RULESET'}) { # Skip whitelist rules file. next if( $file eq "whitelist.rules"); + # Splitt vendor from filename. + my @filename_parts = split(/-/, $file); + + # Assign vendor name for easy processing. + my $vendor = @filename_parts[0]; + + # Skip rulefile if the provider is disabled. + next unless ($vendor ~~ @enabled_providers); + # Call subfunction to read-in rulefile and add rules to # the idsrules hash. &readrulesfile("$file"); @@ -281,17 +294,20 @@ if ($cgiparams{'RULESET'}) { closedir(DIR); - # Gather used rulefiles. - my @used_rulesfiles = &IDS::get_used_rulesfiles(); + # Loop through the array of used providers. + foreach my $provider (@enabled_providers) { + # Gather used rulefiles. + my @used_rulesfiles = &IDS::read_used_provider_rulesfiles($provider); - # Loop through the array of used rulesfiles. - foreach my $rulesfile (@used_rulesfiles) { - # Check if the current rulefile exists in the %idsrules hash. - # If not, the file probably does not exist anymore or contains - # no rules. - if($idsrules{$rulefile}) { - # Add the rulefile state to the %idsrules hash. - $idsrules{$rulefile}{'Rulefile'}{'State'} = "on"; + # Loop through the array of used rulesfiles. + foreach my $rulefile (@used_rulesfiles) { + # Check if the current rulefile exists in the %idsrules hash. + # If not, the file probably does not exist anymore or contains + # no rules. + if($idsrules{$rulefile}) { + # Add the rulefile state to the %idsrules hash. + $idsrules{$rulefile}{'Rulefile'}{'State'} = "on"; + } } } } @@ -479,8 +495,40 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Close file for disabled_sids after writing. close(DISABLED_FILE); + # Handle enabled / disabled rulefiles. + # + # Get enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + + # Loop through the array of enabled providers. + foreach my $provider(@enabled_providers) { + # Array to store the rulefiles which belong to the current processed provider. + my @provider_rulefiles = (); + + # Loop through the array of enabled rulefiles. + foreach my $rulesfile (@enabled_rulefiles) { + # Split the rulefile name. + my @filename_parts = split(/-/, "$rulesfile"); + + # Assign vendor name for easy processings. + my $vendor = @filename_parts[0]; + + # Check if the rulesvendor is our current processed enabled provider. + if ("$vendor" eq "$provider") { + # Add the rulesfile to the array of provider rulesfiles. + push(@provider_rulefiles, $rulesfile); + } + + # Check if any rulesfiles have been found for this provider. + if (@provider_rulefiles) { + # Call function and write the providers used rulesfile file. + &IDS::write_used_provider_rulefiles_file($provider, @provider_rulefiles); + } + } + } + # Call function to generate and write the used rulefiles file. - &IDS::write_used_rulefiles_file(@enabled_rulefiles); + &IDS::write_main_used_rulefiles_file(@enabled_providers); # Lock the webpage and print message. &working_notice("$Lang::tr{'ids apply ruleset changes'}"); @@ -779,6 +827,10 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Cleanup temporary directory. &IDS::cleanup_tmp_directory(); + # Create new empty file for used rulefiles + # for this provider. + &IDS::write_used_provider_rulefiles_file($provider); + # Perform a reload of the page. &reload(); } From a2b4488ae53c92b6ffefa2abb2ee4601e4907014 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 11:46:11 +0200 Subject: [PATCH 059/140] ids.cgi: Finish code to handle toggeling a provider enabled/disabled. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 908ad9c28..e927fb617 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -871,14 +871,28 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Write the changed hash to the providers settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); - # XXX - The ruleset needs to be regenerated - # XXX - Suricata requires a reload or if the last provider - # has been disabled suricata needs to be stopped. + # Get all enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + + # Write the main providers include file. + &IDS::write_main_used_rulefiles_file(@enabled_providers); + # Check if the IDS is running. - #if(&IDS::ids_is_running()) { - # # Call suricatactrl to perform a reload. - # &IDS::call_suricatactrl("reload"); - #} + if(&IDS::ids_is_running()) { + # Gather the amount of enabled providers (elements in the array). + my $amount = @enabled_providers; + + # Check if there are still enabled ruleset providers. + if ($amount >= 1) { + # Call suricatactrl to perform a restart. + &IDS::call_suricatactrl("restart"); + + # No active ruleset provider, suricata has to be stopped. + } else { + # Stop suricata. + &IDS::call_suricatactrl("stop"); + } + } # Undefine providers flag. undef($cgiparams{'PROVIDERS'}); From 2fded6d2ad80a05f87ae895deadc58de05073a34 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 11:50:44 +0200 Subject: [PATCH 060/140] ids.cgi: Finish code to handle the removal of a provider from the list. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index e927fb617..ca58e6ab6 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -906,6 +906,9 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Read-in provider settings file. &General::readhasharray($IDS::providers_settings_file, \%used_providers); + # Grab the provider name bevore deleting it from hash. + my $provider = $used_providers{$cgiparams{'ID'}}[0]; + # Drop entry from the hash. delete($used_providers{$cgiparams{'ID'}}); @@ -915,15 +918,40 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { # Write the changed hash to the provide settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); - # XXX - The ruleset of the provider needs to be dropped. - # XXX - The remain rulest of suricata needs to be regenerated. - # XXX - Suricata requires a reload or if the last provider has - # been removed it has to be stopped. + # Drop the stored ruleset file. + &IDS::drop_dl_rulesfile($provider); + + # Get the name of the provider rulessets include file. + my $provider_used_rulefile = &get_used_provider_rulesfile_file($provider); + + # Drop the file, it is not longer needed. + unlink("$provider_used_rulefile"); + + # Regenerate ruleset. + &IDS::oinkmaster(); + + # Gather all enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + + # Regenerate main providers include file. + &IDS::write_main_used_rulefiles_file(@enabled_providers); + # Check if the IDS is running. - #if(&IDS::ids_is_running()) { - # Call suricatactrl to perform a reload. - # &IDS::call_suricatactrl("reload"); - #} + if(&IDS::ids_is_running()) { + # Get amount of enabled providers. + my $amount = @enabled_providers; + + # Check if at least one enabled provider remains. + if ($amount >= 1) { + # Call suricatactrl to perform a reload. + &IDS::call_suricatactrl("restart"); + + # Stop suricata if no enabled provider remains. + } else { + # Call suricatactrel to perform the stop. + &IDS::call_suricatactrl("stop"); + } + } # Undefine providers flag. undef($cgiparams{'PROVIDERS'}); From 0943ad8c3fa747c701f25f527824db3f1c6de501 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 11:55:40 +0200 Subject: [PATCH 061/140] ids.cgi: Drop old code to handle the settings of the ruleset section. This entirely has been replaced by the providers section and the code to handle the actions of this section. Therefore this code is not longer needed. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 78 +------------------------------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index ca58e6ab6..afc01e26c 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -312,84 +312,8 @@ if ($cgiparams{'RULESET'}) { } } -# Save ruleset configuration. -if ($cgiparams{'RULESET'} eq $Lang::tr{'save'}) { - my %oldsettings; - - # Read-in current (old) IDS settings. - &General::readhash("$IDS::rules_settings_file", \%oldsettings); - - # Prevent form name from been stored in conf file. - delete $cgiparams{'RULESET'}; - - # Check if the choosen vendor (URL) requires an subscription/oinkcode. - if ($IDS::Ruleset::Providers{$cgiparams{'RULES'}}{'requires_subscription'} eq "True") { - # Check if an subscription/oinkcode has been provided. - if ($cgiparams{'OINKCODE'}) { - # Check if the oinkcode contains unallowed chars. - unless ($cgiparams{'OINKCODE'} =~ /^[a-z0-9]+$/) { - $errormessage = $Lang::tr{'invalid input for oink code'}; - } - } else { - # Print an error message, that an subsription/oinkcode is required for this - # vendor. - $errormessage = $Lang::tr{'ids oinkcode required'}; - } - } - - # Go on if there are no error messages. - if (!$errormessage) { - # Store settings into settings file. - &General::writehash("$IDS::rules_settings_file", \%cgiparams); - - # Check if a ruleset is present - if not or the source has been changed download it. - if((! %idsrules) || ($oldsettings{'RULES'} ne $cgiparams{'RULES'})) { - # Check if the red device is active. - unless (-e "${General::swroot}/red/active") { - $errormessage = "$Lang::tr{'could not download latest updates'} - $Lang::tr{'system is offline'}"; - } - - # Check if enough free disk space is availabe. - if(&IDS::checkdiskspace()) { - $errormessage = "$Lang::tr{'not enough disk space'}"; - } - - # Check if any errors happend. - unless ($errormessage) { - # Lock the webpage and print notice about downloading - # a new ruleset. - &working_notice("$Lang::tr{'ids working'}"); - - # Write the modify sid's file and pass the taken ruleaction. - &IDS::write_modify_sids_file(); - - # Call subfunction to download the ruleset. - if(&IDS::downloadruleset()) { - $errormessage = $Lang::tr{'could not download latest updates'}; - - # Call function to store the errormessage. - &IDS::_store_error_message($errormessage); - } else { - # Call subfunction to launch oinkmaster. - &IDS::oinkmaster(); - } - - # Check if the IDS is running. - if(&IDS::ids_is_running()) { - # Call suricatactrl to stop the IDS - because of the changed - # ruleset - the use has to configure it before suricata can be - # used again. - &IDS::call_suricatactrl("stop"); - } - - # Perform a reload of the page. - &reload(); - } - } - } - # Save ruleset. -} elsif ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { +if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Arrays to store which rulefiles have been enabled and will be used. my @enabled_rulefiles; From 106f00bdbb26b9f84300c79f5b7f28dfb2395fcf Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 12:02:45 +0200 Subject: [PATCH 062/140] ids.cgi: Lock the CGI when a provder will be deleted. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index afc01e26c..2ebc78f78 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -842,6 +842,9 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Write the changed hash to the provide settings file. &General::writehasharray($IDS::providers_settings_file, \%used_providers); + # Lock the webpage and print message. + &working_notice("$Lang::tr{'ids apply ruleset changes'}"); + # Drop the stored ruleset file. &IDS::drop_dl_rulesfile($provider); @@ -879,6 +882,9 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Undefine providers flag. undef($cgiparams{'PROVIDERS'}); + + # Reload page. + &reload(); } &Header::openpage($Lang::tr{'intrusion detection system'}, 1, ''); From 4b6cf2a54ab2cbda7d688f4c9cdb45051e354f09 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 15:09:59 +0200 Subject: [PATCH 063/140] ids.cgi: Fix check and message when trying to enable suricata without any enabled or no provider. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 7 +++++-- langs/de/cgi-bin/de.pl | 2 +- langs/en/cgi-bin/en.pl | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 2ebc78f78..4ed2c6fa0 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -530,9 +530,12 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Check if the IDS should be enabled. if ($cgiparams{'ENABLE_IDS'} eq "on") { + # Get enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + # Check if any ruleset is available. Otherwise abort and display an error. - unless(%used_providers) { - $errormessage = $Lang::tr{'ids no ruleset available'}; + unless(@enabled_providers) { + $errormessage = $Lang::tr{'ids no enabled ruleset provider'}; } # Loop through the array of available interfaces. diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 3a4a52e5f..7fbd33073 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1388,7 +1388,7 @@ 'ids monitor traffic only' => 'Netzwerkpakete nur überprüfen (nicht verwerfen)', 'ids monitored interfaces' => 'Überwachte Netzwerkzonen', 'ids no network zone' => 'Bitte wählen Sie mindestens eine zu überwachende Netzwerkzone aus', -'ids no ruleset available' => 'Es ist kein Regelsatz verfügbar. Bitte laden Sie einen Regelsatz herunter.', +'ids no enabled ruleset provider' => 'Es ist kein aktivierter Provider verfügbar. Bitte aktivieren Sie einen oder fügen Sie einen Provider hinzu.', 'ids oinkcode required' => 'Für den ausgewählten Regelsatz wird ein Abonnement oder ein Oinkcode benötigt', 'ids provider' => 'Regelset-Anbieter', 'ids provider settings' => 'Regelset-Anbieter-Einstellungen', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index a62ef0382..4398d28cd 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1418,7 +1418,7 @@ 'ids monitor traffic only' => 'Monitor traffic only', 'ids monitored interfaces' => 'Monitored Interfaces', 'ids no network zone' => 'Please select at least one network zone to be monitored', -'ids no ruleset available' => 'No ruleset is available. Please download one first', +'ids no enabled ruleset provider' => 'No enabled ruleset is available. Please activate or add one first.', 'ids subscription code required' => 'The selected ruleset requires a subscription code', 'ids provider' => 'Provider', 'ids provider settings' => 'Provider settings', From 1fa187335bb641d1f5ae698b21ef7783b228b8e4 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 15:13:30 +0200 Subject: [PATCH 064/140] ids.cgi: Add hardcoded error message to language files. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 3 +-- langs/de/cgi-bin/de.pl | 1 + langs/en/cgi-bin/en.pl | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 4ed2c6fa0..c13a4874c 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -674,9 +674,8 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { foreach my $id ( keys %used_providers) { # Check if the choosen provider is already in use. if ($used_providers{$id}[0] eq "$provider") { - # XXX - add to language file. # Assign error message. - $errormessage = "The choosen provider is already in use."; + $errormessage = "$Lang::tr{'ids the choosen provider is already in use'}"; } } } diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 7fbd33073..c2e661ac5 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1396,6 +1396,7 @@ 'ids ruleset autoupdate in progress' => 'Der Regelsatz wird gerade aktualisiert. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids ruleset settings' => 'Regelsatzeinstellungen', 'ids show' => 'Anzeigen', +'ids the choosen provider is already in use' => 'Der gewhählte Provider wird bereits verwendet.', 'ids visit provider website' => 'Anbieter-Webseite besuchen', 'ids working' => 'Änderungen werden übernommen. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde.', 'iface' => 'Iface', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index 4398d28cd..38ffcfbe2 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1426,6 +1426,7 @@ 'ids ruleset autoupdate in progress' => 'Ruleset update in progress. Please wait until all operations have completed successfully...', 'ids ruleset settings' => 'Ruleset Settings', 'ids show' => 'Show', +'ids the choosen provider is already in use' => 'The choosen provider is already in use.', 'ids visit provider website' => 'Visit provider website', 'ids working' => 'Changes are being applied. Please wait until all operations have completed successfully...', 'iface' => 'Iface', From 02fee15e0e004eda37f194d9a01186f4a1ad4372 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Thu, 1 Apr 2021 15:48:44 +0200 Subject: [PATCH 065/140] ids.cgi: Prevent from chainging the provider when editing an existing one. This commit locks the dropdown menu for selecting a provider, in case an existing one should be edited. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index c13a4874c..ec92c0f96 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1621,9 +1621,21 @@ print < - \n"; + } + + print " + + + + + + +END +; + &Header::closebox(); +} + # ## A function to display a notice, to lock the webpage and ## tell the user which action currently will be performed. diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index c2e661ac5..72efbba37 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1375,11 +1375,13 @@ 'ids add provider' => 'Provider hinzufügen', 'ids apply' => 'Übernehmen', 'ids apply ruleset changes' => 'Regeländerungen werden übernommen. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', +'ids autoupdates' => 'Automatische Updates', 'ids automatic rules update' => 'Automatische Regelaktualisierung', 'ids customize ruleset' => 'Regelset anpassen', 'ids download new ruleset' => 'Das neue Regelsatz wird heruntergeladen und entpackt. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids enable' => 'Einbruchsverhinderungssystem aktivieren', 'ids enable automatic updates' => 'Automatische Updates aktivieren', +'ids force ruleset update' => 'Regelset jetzt aktualisieren', 'ids hide' => 'Verstecken', 'ids ignored hosts' => 'Ausnahmeliste', 'ids log hits' => 'Gesamtanzahl der Regeltreffer für', @@ -1392,6 +1394,7 @@ 'ids oinkcode required' => 'Für den ausgewählten Regelsatz wird ein Abonnement oder ein Oinkcode benötigt', 'ids provider' => 'Regelset-Anbieter', 'ids provider settings' => 'Regelset-Anbieter-Einstellungen', +'ids reset provider' => 'Providereinstellungen zurücksetzen', 'ids rules update' => 'Regelsatz', 'ids ruleset autoupdate in progress' => 'Der Regelsatz wird gerade aktualisiert. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids ruleset settings' => 'Regelsatzeinstellungen', diff --git a/langs/en/cgi-bin/en.pl b/langs/en/cgi-bin/en.pl index 38ffcfbe2..59496b195 100644 --- a/langs/en/cgi-bin/en.pl +++ b/langs/en/cgi-bin/en.pl @@ -1410,6 +1410,7 @@ 'ids download new ruleset' => 'Downloading and unpacking new ruleset. Please wait until all operations have completed successfully...', 'ids enable' => 'Enable Intrusion Prevention System', 'ids enable automatic updates' => 'Enable automatic updates', +'ids force ruleset update' => 'Force ruleset update', 'ids hide' => 'Hide', 'ids ignored hosts' => 'Whitelisted Hosts', 'ids log hits' => 'Total of number of activated rules for', @@ -1422,6 +1423,7 @@ 'ids subscription code required' => 'The selected ruleset requires a subscription code', 'ids provider' => 'Provider', 'ids provider settings' => 'Provider settings', +'ids reset provider' => 'Reset provider', 'ids rules update' => 'Ruleset', 'ids ruleset autoupdate in progress' => 'Ruleset update in progress. Please wait until all operations have completed successfully...', 'ids ruleset settings' => 'Ruleset Settings', From f3d421a3b183802c7d4af1333d73057964d18869 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 11:00:17 +0200 Subject: [PATCH 079/140] ids.cgi: Make backend code for forced ruleset update working again. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index e702e8d2e..1684b7757 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -496,7 +496,10 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &reload(); # Download new ruleset. -} elsif ($cgiparams{'RULESET'} eq $Lang::tr{'update ruleset'}) { +} elsif ($cgiparams{'PROVIDERS'} eq $Lang::tr{'ids force ruleset update'}) { + # Assign given provider handle. + my $provider = $cgiparams{'PROVIDER'}; + # Check if the red device is active. unless (-e "${General::swroot}/red/active") { $errormessage = "$Lang::tr{'could not download latest updates'} - $Lang::tr{'system is offline'}"; @@ -514,7 +517,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &working_notice("$Lang::tr{'ids download new ruleset'}"); # Call subfunction to download the ruleset. - if(&IDS::downloadruleset()) { + if(&IDS::downloadruleset($provider)) { $errormessage = $Lang::tr{'could not download latest updates'}; # Call function to store the errormessage. From 515a694d1c8bcd90bf52f35dc59d4990ff6b4935 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 11:21:00 +0200 Subject: [PATCH 080/140] ids.cgi: Add code to handle the reset of a provider to it's defaults. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 1684b7757..47e9ac778 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -539,6 +539,61 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { &reload(); } } + +# Reset a provider to it's defaults. +} elsif ($cgiparams{'PROVIDERS'} eq "$Lang::tr{'ids reset provider'}") { + # Grab provider handle from cgihash. + my $provider = $cgiparams{'PROVIDER'}; + + # Lock the webpage and print message. + &working_notice("$Lang::tr{'ids apply ruleset changes'}"); + + # Create new empty file for used rulefiles + # for this provider. + &IDS::write_used_provider_rulefiles_file($provider); + + # Call function to get the path and name for the given providers + # oinkmaster modified sids file. + my $provider_modified_sids_file = &IDS::get_oinkmaster_provider_modified_sids_file($provider); + + # Check if the file exists. + if (-f $provider_modified_sids_file) { + # Remove the file, as requested. + unlink("$provider_modified_sids_file"); + } + + # Alter the oinkmaster provider includes file and remove the provider. + &IDS::alter_oinkmaster_provider_includes_file("remove", $provider); + + # Regenerate ruleset. + &IDS::oinkmaster(); + + # Check if the IDS is running. + if(&IDS::ids_is_running()) { + # Get enabled providers. + my @enabled_providers = &IDS::get_enabled_providers(); + + # Get amount of enabled providers. + my $amount = @enabled_providers; + + # Check if at least one enabled provider remains. + if ($amount >= 1) { + # Call suricatactrl to perform a reload. + &IDS::call_suricatactrl("restart"); + + # Stop suricata if no enabled provider remains. + } else { + # Call suricatactrel to perform the stop. + &IDS::call_suricatactrl("stop"); + } + } + + # Undefine providers flag. + undef($cgiparams{'PROVIDERS'}); + + # Reload page. + &reload(); + # Save IDS settings. } elsif ($cgiparams{'IDS'} eq $Lang::tr{'save'}) { my %oldidssettings; From 71766c081c7f3a9a8a7c2356d4ac2f23eae27913 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 11:24:03 +0200 Subject: [PATCH 081/140] langs-de.pl: Fix grammar. Even as a native speaker, it seems german sometimes is a very difficult language...... Signed-off-by: Stefan Schantl --- langs/de/cgi-bin/de.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langs/de/cgi-bin/de.pl b/langs/de/cgi-bin/de.pl index 72efbba37..a4a81fc38 100644 --- a/langs/de/cgi-bin/de.pl +++ b/langs/de/cgi-bin/de.pl @@ -1378,7 +1378,7 @@ 'ids autoupdates' => 'Automatische Updates', 'ids automatic rules update' => 'Automatische Regelaktualisierung', 'ids customize ruleset' => 'Regelset anpassen', -'ids download new ruleset' => 'Das neue Regelsatz wird heruntergeladen und entpackt. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', +'ids download new ruleset' => 'Das neue Regelset wird heruntergeladen und entpackt. Bitte warten Sie, bis dieser Vorgang erfolgreich beendet wurde...', 'ids enable' => 'Einbruchsverhinderungssystem aktivieren', 'ids enable automatic updates' => 'Automatische Updates aktivieren', 'ids force ruleset update' => 'Regelset jetzt aktualisieren', From 6875f9ce7c014a9236a1c523f14381a30e1972eb Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 11:45:00 +0200 Subject: [PATCH 082/140] update-ids-ruleset: Port script to work with multiple providers. Signed-off-by: Stefan Schantl --- src/scripts/update-ids-ruleset | 43 ++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/scripts/update-ids-ruleset b/src/scripts/update-ids-ruleset index dbe5b6849..3a3f17443 100644 --- a/src/scripts/update-ids-ruleset +++ b/src/scripts/update-ids-ruleset @@ -2,7 +2,7 @@ ############################################################################### # # # IPFire.org - A linux based firewall # -# Copyright (C) 2018 IPFire Team # +# Copyright (C) 2018-2021 IPFire Team # # # # This program is free software: you can redistribute it and/or modify # # it under the terms of the GNU General Public License as published by # @@ -26,6 +26,9 @@ require '/var/ipfire/general-functions.pl'; require "${General::swroot}/ids-functions.pl"; require "${General::swroot}/lang.pl"; +# Hash to store the configured providers. +my %providers = (); + # The user and group name as which this script should be run. my $run_as = 'nobody'; @@ -63,21 +66,37 @@ if(&IDS::checkdiskspace()) { # Lock the IDS page. &IDS::lock_ids_page(); -# Call the download function and gather the new ruleset. -if(&IDS::downloadruleset()) { - # Store error message for displaying in the WUI. - &IDS::_store_error_message("$Lang::tr{'could not download latest updates'}"); +# Grab the configured providers. +&General::readhasharray("$IDS::providers_settings_file", \%providers); - # Unlock the IDS page. - &IDS::unlock_ids_page(); +# Loop through the array of available providers. +foreach my $id (keys %providers) { + # Assign some nice variabled. + my $provider = $providers{$id}[0]; + my $autoupdate_status = $providers{$id}[3]; - # Exit. - exit 0; + # Skip the provider if autoupdate is not enabled. + next unless($autoupdate_status eq "enabled"); + + # Call the download function and gather the new ruleset for the current processed provider. + if(&IDS::downloadruleset($provider)) { + # Store error message for displaying in the WUI. + &IDS::_store_error_message("$provider: $Lang::tr{'could not download latest updates'}"); + + # Unlock the IDS page. + &IDS::unlock_ids_page(); + + # Exit. + exit 0; + } + + # Get path and name of the stored rules file or archive. + my $stored_file = &IDS::_get_dl_rulesfile($provider); + + # Set correct ownership for the downloaded tarball. + &IDS::set_ownership("$stored_file"); } -# Set correct ownership for the downloaded tarball. -&IDS::set_ownership("$IDS::rulestarball"); - # Call oinkmaster to alter the ruleset. &IDS::oinkmaster(); From 35bc92a30717461a53d070b7a2d49ddcdc1c65ba Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 13:46:47 +0200 Subject: [PATCH 083/140] ids-functions.pl: Fix accidently commited debug file path. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index db5ef4666..f95a02ddd 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -27,7 +27,7 @@ package IDS; require '/var/ipfire/general-functions.pl'; require "${General::swroot}/network-functions.pl"; -require "${General::swroot}/suricata/ruleset-sources-new"; +require "${General::swroot}/suricata/ruleset-sources"; # Location where all config and settings files are stored. our $settingsdir = "${General::swroot}/suricata"; From 01fc880cf3d6a19d1c2809f9adecd78278ebb49a Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 13:47:36 +0200 Subject: [PATCH 084/140] ids-functions.pl: Only read providers used rulefiles file if it exists. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index f95a02ddd..7a3661540 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1694,8 +1694,8 @@ sub read_used_provider_rulesfiles($) { # Get the used rulesefile file for the provider. my $rulesfile_file = &get_used_provider_rulesfile_file($provider); - # Check if the used rulesfile is empty. - unless (-z $rulesfile_file) { + # Check if the a used rulesfile exists for this provider. + if (-f $rulesfile_file) { # Open the file or used rulefiles and read-in content. open(FILE, $rulesfile_file) or die "Could not open $rulesfile_file. $!\n"; From 1b5aec1b7db40749fb0313f74e2670fc99a891cd Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Fri, 2 Apr 2021 20:22:15 +0200 Subject: [PATCH 085/140] ids-functions.pl: Move code to handle plain rules files to extractruleset() function. Now everithing which is extracting or moving stored ruleset files is easily accessing via one function which takes care about. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 150 ++++++++++++++++---------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index 7a3661540..f4f06413c 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -460,6 +460,9 @@ sub downloadruleset ($) { # ## Function to extract a given ruleset. +## +## In case the ruleset provider offers a plain file, it simply will +## be copied. # sub extractruleset ($) { my ($provider) = @_; @@ -470,12 +473,15 @@ sub extractruleset ($) { # Load perl module to deal with files and path. use File::Basename; + # Load perl module for file copying. + use File::Copy; + # Get full path and downloaded rulesfile for the given provider. my $tarball = &_get_dl_rulesfile($provider); # Check if the file exists. unless (-f $tarball) { - &_log_to_syslog("Could not extract ruleset file: $tarball"); + &_log_to_syslog("Could not find ruleset file: $tarball"); # Return nothing. return; @@ -486,69 +492,84 @@ sub extractruleset ($) { mkdir("$tmp_rules_directory") unless (-d "$tmp_rules_directory"); mkdir("$tmp_conf_directory") unless (-d "$tmp_conf_directory"); - # Initialize the tar module. - my $tar = Archive::Tar->new($tarball); + # Omit the type (dl_type) of the stored ruleset. + my $type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; - # Get the filelist inside the tarball. - my @packed_files = $tar->list_files; + # Handle the different ruleset types. + if ($type eq "plain") { + # Generate destination filename an full path. + my $destination = "$tmp_rules_directory/$provider\-ruleset.rules"; - # Loop through the filelist. - foreach my $packed_file (@packed_files) { - my $destination; + # Copy the file into the temporary rules directory. + copy($tarball, $destination); - # Splitt the packed file into chunks. - my $file = fileparse($packed_file); + } elsif ( $type eq "archive") { + # Initialize the tar module. + my $tar = Archive::Tar->new($tarball); - # Handle msg-id.map file. - if ("$file" eq "sid-msg.map") { - # Set extract destination to temporary config_dir. - $destination = "$tmp_conf_directory/$provider\-sid-msg.map"; - # Handle classification.conf - } elsif ("$file" eq "classification.config") { - # Set extract destination to temporary config_dir. - $destination = "$tmp_conf_directory/$provider\-classification.config"; - # Handle rules files. - } elsif ($file =~ m/\.rules$/) { - my $rulesfilename; + # Get the filelist inside the tarball. + my @packed_files = $tar->list_files; - # Splitt the filename into chunks. - my @filename = split("-", $file); + # Loop through the filelist. + foreach my $packed_file (@packed_files) { + my $destination; - # Reverse the array. - @filename = reverse(@filename); + # Splitt the packed file into chunks. + my $file = fileparse($packed_file); - # Get the amount of elements in the array. - my $elements = @filename; + # Handle msg-id.map file. + if ("$file" eq "sid-msg.map") { + # Set extract destination to temporary config_dir. + $destination = "$tmp_conf_directory/$provider\-sid-msg.map"; - # Remove last element of the hash. - # It contains the vendor name, which will be replaced. - if ($elements >= 3) { + # Handle classification.conf + } elsif ("$file" eq "classification.config") { + # Set extract destination to temporary config_dir. + $destination = "$tmp_conf_directory/$provider\-classification.config"; + + # Handle rules files. + } elsif ($file =~ m/\.rules$/) { + my $rulesfilename; + + # Splitt the filename into chunks. + my @filename = split("-", $file); + + # Reverse the array. + @filename = reverse(@filename); + + # Get the amount of elements in the array. + my $elements = @filename; + + # Remove last element of the hash. + # It contains the vendor name, which will be replaced. + if ($elements >= 3) { # Remove last element from hash. - pop(@filename); + pop(@filename); + } + + # Check if the last element of the filename does not + # contain the providers name. + if ($filename[-1] ne "$provider") { + # Add provider name as last element. + push(@filename, $provider); + } + + # Reverse the array back. + @filename = reverse(@filename); + + # Generate the name for the rulesfile. + $rulesfilename = join("-", @filename); + + # Set extract destination to temporaray rules_dir. + $destination = "$tmp_rules_directory/$rulesfilename"; + } else { + # Skip all other files. + next; } - # Check if the last element of the filename does not - # contain the providers name. - if ($filename[-1] ne "$provider") { - # Add provider name as last element. - push(@filename, $provider); - } - - # Reverse the array back. - @filename = reverse(@filename); - - # Generate the name for the rulesfile. - $rulesfilename = join("-", @filename); - - # Set extract destination to temporaray rules_dir. - $destination = "$tmp_rules_directory/$rulesfilename"; - } else { - # Skip all other files. - next; + # Extract the file to the temporary directory. + $tar->extract_file("$packed_file", "$destination"); } - - # Extract the file to the temporary directory. - $tar->extract_file("$packed_file", "$destination"); } } @@ -557,9 +578,6 @@ sub extractruleset ($) { ## call the functions to merge the additional config files. (classification, sid-msg, etc.). # sub oinkmaster () { - # Load perl module for file copying. - use File::Copy; - # Check if the files in rulesdir have the correct permissions. &_check_rulesdir_permissions(); @@ -571,26 +589,8 @@ sub oinkmaster () { # Loop through the array of enabled providers. foreach my $provider (@enabled_providers) { - # Omit the type (dl_type) of the stored ruleset. - my $type = $IDS::Ruleset::Providers{$provider}{'dl_type'}; - - # Handle the different ruleset types. - if ($type eq "archive") { - # Call the extractruleset function. - &extractruleset($provider); - } elsif ($type eq "plain") { - # Generate filename and full path for the stored rulesfile. - my $dl_rulesfile = &_get_dl_rulesfile($provider); - - # Generate destination filename an full path. - my $destination = "$tmp_rules_directory/$provider\-ruleset.rules"; - - # Copy the file into the temporary rules directory. - copy($dl_rulesfile, $destination); - } else { - # Skip unknown type. - next; - } + # Call the extractruleset function. + &extractruleset($provider); } # Load perl module to talk to the kernel syslog. From 52071c0e9e7956c2e1a42430c01f06a383c2787d Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 4 Apr 2021 08:15:48 +0200 Subject: [PATCH 086/140] Revert "ids-functions.pl: Remove config files on rulesdir cleanup." Not all config files are shipped by the rulesets. For example the "threshold.conf" and the "referneces.conf" are not include in each ruleset. Therefore it is not a common way to delete all config files. It is much safer to simple keep them and overwrite existing ones by the generated ones. This reverts commit a71c3c9dcc60541aa4504d0f1fb0a78c0d58ed5e. --- config/cfgroot/ids-functions.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index f4f06413c..c231e0d44 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1094,6 +1094,9 @@ sub _cleanup_rulesdir() { # We only want files. next unless (-f "$rulespath/$file"); + # Skip element if it has config as file extension. + next if ($file =~ m/\.config$/); + # Skip rules file for whitelisted hosts. next if ("$rulespath/$file" eq $whitelist_file); From b35e27a28a66410ff0a4c69739b1d98d8bfd2d30 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 4 Apr 2021 08:25:36 +0200 Subject: [PATCH 087/140] backup: Adjust includes file to include new IDS files into backups. Signed-off-by: Stefan Schantl --- config/backup/include | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/backup/include b/config/backup/include index 5db452cda..3b96b1d62 100644 --- a/config/backup/include +++ b/config/backup/include @@ -49,7 +49,7 @@ /var/ipfire/qos/bin/qos.sh /var/ipfire/suricata/*.conf /var/ipfire/suricata/*.yaml -/var/ipfire/suricata/rules-settings +/var/ipfire/suricata/providers-settings /var/ipfire/*/settings /var/ipfire/time/ /var/ipfire/urlfilter @@ -59,4 +59,5 @@ /var/log/rrd/* /var/log/rrd/collectd /var/log/vnstat -/var/tmp/idsrules.tar.gz +/var/tmp/idsrules-*.tar.gz +/var/tmp/idsrules-*.rules From 69b3156f7467fd941e904c5f7316a83bc11636ae Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 4 Apr 2021 10:15:27 +0200 Subject: [PATCH 088/140] IDS: Move read_enabled_disabled_sids_file() function to ids-functions.pl. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 51 +++++++++++++++++++++++++++++++ html/cgi-bin/ids.cgi | 53 +-------------------------------- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index c231e0d44..e305be15f 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -955,6 +955,57 @@ sub alter_oinkmaster_provider_includes_file ($$) { close(FILE); } +# +## Function to read-in the given enabled or disables sids file. +# +sub read_enabled_disabled_sids_file($) { + my ($file) = @_; + + # Temporary hash to store the sids and their state. It will be + # returned at the end of this function. + my %temphash; + + # Open the given filename. + open(FILE, "$file") or die "Could not open $file. $!\n"; + + # Loop through the file. + while() { + # Remove newlines. + chomp $_; + + # Skip blank lines. + next if ($_ =~ /^\s*$/); + + # Skip coments. + next if ($_ =~ /^\#/); + + # Splitt line into sid and state part. + my ($state, $sid) = split(" ", $_); + + # Skip line if the sid is not numeric. + next unless ($sid =~ /\d+/ ); + + # Check if the sid was enabled. + if ($state eq "enablesid") { + # Add the sid and its state as enabled to the temporary hash. + $temphash{$sid} = "enabled"; + # Check if the sid was disabled. + } elsif ($state eq "disablesid") { + # Add the sid and its state as disabled to the temporary hash. + $temphash{$sid} = "disabled"; + # Invalid state - skip the current sid and state. + } else { + next; + } + } + + # Close filehandle. + close(FILE); + + # Return the hash. + return %temphash; +} + # ## Function to check if the IDS is running. # diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 47e9ac778..8ee16930a 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -358,7 +358,7 @@ if ($cgiparams{'RULESET'} eq $Lang::tr{'ids apply'}) { # Check if a modified sids file for this provider exists. if (-f $providers_modified_sids_file) { # Read-in the file for enabled/disabled sids. - %enabled_disabled_sids = &read_enabled_disabled_sids_file($providers_modified_sids_file); + %enabled_disabled_sids = &IDS::read_enabled_disabled_sids_file($providers_modified_sids_file); } # Loop through the hash of idsrules. @@ -1990,57 +1990,6 @@ sub get_memory_usage($) { return; } -# -## Function to read-in the given enabled or disables sids file. -# -sub read_enabled_disabled_sids_file($) { - my ($file) = @_; - - # Temporary hash to store the sids and their state. It will be - # returned at the end of this function. - my %temphash; - - # Open the given filename. - open(FILE, "$file") or die "Could not open $file. $!\n"; - - # Loop through the file. - while() { - # Remove newlines. - chomp $_; - - # Skip blank lines. - next if ($_ =~ /^\s*$/); - - # Skip coments. - next if ($_ =~ /^\#/); - - # Splitt line into sid and state part. - my ($state, $sid) = split(" ", $_); - - # Skip line if the sid is not numeric. - next unless ($sid =~ /\d+/ ); - - # Check if the sid was enabled. - if ($state eq "enablesid") { - # Add the sid and its state as enabled to the temporary hash. - $temphash{$sid} = "enabled"; - # Check if the sid was disabled. - } elsif ($state eq "disablesid") { - # Add the sid and its state as disabled to the temporary hash. - $temphash{$sid} = "disabled"; - # Invalid state - skip the current sid and state. - } else { - next; - } - } - - # Close filehandle. - close(FILE); - - # Return the hash. - return %temphash; -} - # ## Function to get the provider name from the language file or providers file for a given handle. # From 1b0e555fd37ca59847fc6d1714cada26503bb823 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sun, 4 Apr 2021 12:22:13 +0200 Subject: [PATCH 089/140] ids-functions.pl: Only write existing provider specific used rulesfiles files into main include yaml file. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index e305be15f..2b5f9cb10 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -1408,8 +1408,11 @@ sub write_main_used_rulefiles_file (@) { # Call function to get the providers used rulefiles file. my $filename = &get_used_provider_rulesfile_file($provider); - # Print the provider to the file. - print FILE "include\: $filename\n"; + # Check if the file exists and write it into the used rulefiles file. + if (-f $filename) { + # Print the provider to the file. + print FILE "include\: $filename\n"; + } } # Always include the file which hold the static includes. From 4015d3f499c85cece577e360277cdc0e95a3d083 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Mon, 5 Apr 2021 07:11:04 +0200 Subject: [PATCH 090/140] ids.cgi: Sort elements in providers dropdown menu. Signed-off-by: Stefan Schantl --- html/cgi-bin/ids.cgi | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/html/cgi-bin/ids.cgi b/html/cgi-bin/ids.cgi index 8ee16930a..0f039c33c 100644 --- a/html/cgi-bin/ids.cgi +++ b/html/cgi-bin/ids.cgi @@ -1748,10 +1748,23 @@ END } print "