diff --git a/config/cfgroot/ipblocklist-functions.pl b/config/cfgroot/ipblocklist-functions.pl index f7ffbdbca..f312e349a 100644 --- a/config/cfgroot/ipblocklist-functions.pl +++ b/config/cfgroot/ipblocklist-functions.pl @@ -21,13 +21,29 @@ package IPblocklist; -require '/var/ipfire/ipblocklist/sources'; +require '/var/ipfire/general-functions.pl'; +require "${General::swroot}/ipblocklist/sources"; + +# The directory where all ipblocklist related files and settings are stored. +our $settings_dir = "/var/ipfire/ipblocklist"; + +# Main settings file. +our $settings_file = "$settings_dir/settings"; + +# The file which keeps the time, when a blocklist last has been modified. +my $modified_file = "$settings_dir/modified"; # Location where the blocklists in ipset compatible format are stored. -our $blocklist_dir = "/var/lib/ipblocklist"; +my $blocklist_dir = "/var/lib/ipblocklist"; # File extension of the blocklist files. -our $blocklist_file_extension = ".conf"; +my $blocklist_file_extension = ".conf"; + +# Hash which calls the correct parser functions. +my %parsers = ( + 'ip-or-net-list' => \&parse_ip_or_net_list, + 'dshield' => \&parse_dshield +); # ## Function to get all available blocklists. @@ -58,4 +74,187 @@ sub get_ipset_db_file($) { return $file; } +# +## The main download_and_create blocklist function. +## +## Uses LWP to download a given blocklist. The If-Modified-Since header is +## specified in the request so that only updated lists are downloaded (providing +## that the server supports this functionality). +## +## Once downloaded the list gets parsed, converted and stored in an ipset compatible +## format. +## +## Parameters: +## list The name of the blocklist +## +## Returns: +## nothing - On success +## not_modified - In case the servers responds with "Not modified" (304) +## dl_error - If the requested blocklist could not be downloaded. +# +sub download_and_create_blocklist($) { + my ($list) = @_; + + # Check if the given blockist is known and data available. + unless($IPblocklist::List::sources{$list}) { + # No valid data for this blocklist - exit and return "1". + return 1; + } + + # The allowed maximum download size in bytes. + my $max_dl_bytes = 10_485_760; + + # The amount of download attempts before giving up and + # logging an error. + my $max_dl_attempts = 5; + + # Read proxysettings. + my %proxysettings=(); + &General::readhash("${General::swroot}/proxy/settings", \%proxysettings); + + # Load required perl module to handle the download. + use LWP::UserAgent; + + # Create a user agent for downloading the blacklist + # Limit the download size for safety + my $ua = LWP::UserAgent->new ( + ssl_opts => { + SSL_ca_file => '/etc/ssl/cert.pem', + verify_hostname => 1, + }, + + max_size => $max_dl_bytes, + ); + + # Set timeout to 10 seconds. + $ua->timeout(10); + + # Check if an upstream proxy is configured. + if ($proxysettings{'UPSTREAM_PROXY'}) { + my $proxy_url; + + $proxy_url = "http://"; + + # Check if the proxy requires authentication. + if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) { + $proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@"; + } + + # Add proxy server address and port. + $proxy_url .= $proxysettings{'UPSTREAM_PROXY'}; + + # Setup proxy settings. + $ua->proxy(['http', 'https'], $proxy_url); + } + + # Gather the details, when a list got modified last time. + my %modified = (); + + # Read-in data if the file exists. + &General::readhash($modified_file, \%modified ) if (-e $modified_file); + + # Get the last modified time for this list. + my $last_modified = gmtime($modified{$list} || 0); + + my $dl_attempt = 1; + my $response; + + # Download and rety on failure loop. + while ($dl_attempt <= $max_dl_attempts) { + # Try to determine if there is a newer blocklist since last time and grab it. + $response = $ua->get($IPblocklist::List::sources{$list}{'url'}, 'If-Modified-Since' => $last_modified ); + + # Check if the download attempt was successfull. + if ($response->is_success) { + # We successfully grabbed the list - no more retries needed, break the loop. + # Further process the script code. + last; + + # Exit, if the server responds with "Not modified (304). + } elsif ($response->code == 304) { + # Exit and return "not modified". + return "not_modified"; + + # Exit and log an erro + } elsif ($dl_attempt eq $max_dl_attempts) { + # Exit and return "dl_error". + return "dl_error"; + } + + # Increase download attempt counter. + $dl_attempt++; + } + + # Update the timestamp for the new or modified list. + $modified{$list} = $response->last_modified; + + # Write-back the modified timestamps. + &General::writehash($modified_file, \%modified); + + # Parse and loop through the downloaded list. + my @blocklist = (); + + # Get the responsible parser for the current list. + my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}}; + + # Loop through the grabbed raw list. + foreach my $line (split /[\r\n]+/, $response->content) { + # Remove newlines. + chomp $line; + + # Call the parser and obtain the addess or network. + my $address = &$parser($line); + + # Skip the line if it does not contain an address. + next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/); + + # Check if we got a single address. + if ($address =~ m|/32|) { + # Add /32 as prefix. + $address =~ s|/32||; + } + + # Push the address/network to the blocklist array. + push(@blocklist, $address); + } + + # Get amount of entries in the blocklist array. + my $list_entries = scalar(@blocklist); + + # Optain the filename for this blocklist to save. + my $file = &get_ipset_db_file($list); + + # Open the file for writing. + open(FILE, ">", "$file") or die "Could not write to $file. $!\n"; + + # Write file header. + print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n"; + + # Calculate the hashsize for better list performance. + my $hashsize = &_calculate_hashsize($list_entries); + + # Simply set the limit of list elements to the double of current list elements. + my $maxelem = $list_entries *2; + + # Write line to create the set. + # + # We safely can use hash:net as type because it supports single addresses and networks. + print FILE "create $list hash:net family inet hashsize $hashsize maxelem $maxelem -exist\n"; + + # Write line to flush the set itself during loading. + print FILE "flush $list\n"; + + # Loop through the array which contains the blocklist. + foreach my $entry (@blocklist) { + # Add the entry to the list. + print FILE "add $list $entry\n"; + } + + # Close the file handle. + close(FILE); + + # Finished. + return; +} + 1;