mirror of
https://github.com/vincentmli/bpfire.git
synced 2026-04-25 10:22:59 +02:00
524dae818f
We do not deliver anything via HTTP or FTP any more and therefore nothing can be cached any more. Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
436 lines
13 KiB
Perl
436 lines
13 KiB
Perl
#!/usr/bin/perl
|
|
#
|
|
# This code is distributed under the terms of the GPL
|
|
#
|
|
# (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
|
|
#
|
|
# Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
|
|
#
|
|
# $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
|
|
#
|
|
# ChangeLog:
|
|
#
|
|
# 2012-10-26: nightshift - move curly bracket to capture AVG download source.
|
|
# - Adding BIG HINT for new update source#
|
|
#
|
|
|
|
use strict;
|
|
use HTTP::Date;
|
|
|
|
$|=1;
|
|
|
|
my $swroot="/var/ipfire";
|
|
my $updcachedir="/var/updatecache";
|
|
my $apphome="/var/ipfire/updatexlrator";
|
|
my $logfile="/var/log/updatexlrator/cache.log";
|
|
my $wget="/usr/bin/wget";
|
|
my $debug=(-e "$apphome/debug");
|
|
my $http_port='81';
|
|
my %netsettings=();
|
|
my %proxysettings=();
|
|
my %xlratorsettings=();
|
|
my $logging=0;
|
|
my $passive_mode=0;
|
|
my $maxusage=0;
|
|
my $nice='';
|
|
my @tmp=();
|
|
my $request='';
|
|
my $xlrator_url=0;
|
|
my $source_url='';
|
|
my $hostaddr='';
|
|
my $username='';
|
|
my $method='';
|
|
my $unique = 0;
|
|
my $mirror = 1;
|
|
|
|
readhash("${swroot}/ethernet/settings", \%netsettings);
|
|
|
|
if (-e "$swroot/updatexlrator/settings")
|
|
{
|
|
&readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
|
|
if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
|
|
if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
|
|
$maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
|
|
if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
|
|
}
|
|
if (!$maxusage) { $maxusage=75; };
|
|
|
|
# dotzball: check for dead downloads
|
|
system("$apphome/bin/checkdeaddl &");
|
|
|
|
while (<>) {
|
|
|
|
$request=$_;
|
|
|
|
@tmp=split(/ /,$request);
|
|
chomp(@tmp);
|
|
|
|
$source_url = $tmp[0];
|
|
$hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
|
|
$username = $tmp[2]; if ($username eq '') { $username='-'; };
|
|
$method = $tmp[3];
|
|
|
|
$xlrator_url = "";
|
|
|
|
if (($method eq 'GET') || ($method eq 'HEAD'))
|
|
{
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Windows Update / Windows Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if (
|
|
(($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i) ||
|
|
($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i))
|
|
&& ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
|
|
&& ($source_url !~ m@\&@)
|
|
)
|
|
{
|
|
# NOTE: check_cache will change to $mirror instead of $unique if the filename contains an SHA1 hash
|
|
# and the URL is not found in cache!
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Adobe Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
|
|
if (
|
|
($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|ms[ip]|bin|dmg|idx|gz)$@i) ||
|
|
($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|ms[ip]|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
|
|
($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
|
|
)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Linux Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if (
|
|
($source_url =~ m@^[h|f]t?tp://[^?]+\.(pkg\.tar\.xz|deb|rpm)$@i) ||
|
|
($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tar\.xz|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i)
|
|
)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Trend Micro Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if (
|
|
($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
|
|
($source_url !~ m@.*/tmnotify\.dat$@i) &&
|
|
($source_url !~ m@.*/ini_xml\.zip$@i) &&
|
|
($source_url !~ m@.*/server\.ini$@i)
|
|
)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Symantec Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Apple Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if (
|
|
(($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
|
|
($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
|
|
($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
|
|
)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Avast Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if ($source_url =~ m@^http://(ion|download)[\d]+\.avast\.com/.*\.(exe|vpu|vpx)$@i)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Mozilla Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if ($source_url=~ m@^http://.*\.mozilla\.net/.*\.((complete|partial)\.mar|exe)$@i)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Mcafee
|
|
# -----------------------------------------------------------
|
|
|
|
if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs|z|gem|dat|zip)$@i)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: Avira Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
if (
|
|
($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) ||
|
|
($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) ||
|
|
($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i)
|
|
)
|
|
{
|
|
$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
|
|
}
|
|
|
|
# -----------------------------------------------------------
|
|
# Section: AVG Downloads
|
|
# -----------------------------------------------------------
|
|
|
|
# if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
|
|
# {
|
|
# $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
|
|
# }
|
|
|
|
# ----------- ADD NEW SOURCES BEFORE THIS LINE !!! ------------------
|
|
}
|
|
|
|
if ($xlrator_url) {
|
|
print "OK rewrite-url=\"$xlrator_url\"\n";
|
|
} else {
|
|
print "ERR\n";
|
|
}
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub readhash
|
|
{
|
|
my $filename = $_[0];
|
|
my $hash = $_[1];
|
|
my ($var, $val);
|
|
|
|
if (-e $filename)
|
|
{
|
|
open(FILE, $filename) or die "Unable to read file $filename";
|
|
while (<FILE>)
|
|
{
|
|
chop;
|
|
($var, $val) = split /=/, $_, 2;
|
|
if ($var)
|
|
{
|
|
$val =~ s/^\'//g;
|
|
$val =~ s/\'$//g;
|
|
|
|
# Untaint variables read from hash
|
|
$var =~ /([A-Za-z0-9_-]*)/; $var = $1;
|
|
$val =~ /([\w\W]*)/; $val = $1;
|
|
$hash->{$var} = $val;
|
|
}
|
|
}
|
|
close FILE;
|
|
}
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub writelog
|
|
{
|
|
if ($logging)
|
|
{
|
|
open(LOGFILE,">>$logfile");
|
|
print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
|
|
close(LOGFILE);
|
|
}
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub debuglog
|
|
{
|
|
if ($debug)
|
|
{
|
|
open(LOGFILE,">>/var/log/updatexlrator/debug.log");
|
|
my @now = localtime(time);
|
|
printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
|
|
close(LOGFILE);
|
|
}
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub setcachestatus
|
|
{
|
|
open (FILE,">>$_[0]");
|
|
print FILE "$_[1]\n";
|
|
close FILE;
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub diskfree
|
|
{
|
|
open(DF,"/bin/df --block-size=1 $_[0]|");
|
|
my @dfdata = <DF>;
|
|
close DF;
|
|
shift(@dfdata);
|
|
chomp(@dfdata);
|
|
my $dfstr = join(' ',@dfdata);
|
|
my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
|
|
if ($free =~ m/^(\d+)$/)
|
|
{
|
|
return $free;
|
|
}
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub diskusage
|
|
{
|
|
open(DF,"/bin/df $_[0]|");
|
|
my @dfdata = <DF>;
|
|
close DF;
|
|
shift(@dfdata);
|
|
chomp(@dfdata);
|
|
my $dfstr = join(' ',@dfdata);
|
|
my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
|
|
if ($percent =~ m/^(\d+)%$/)
|
|
{
|
|
$percent =~ s/%$//;
|
|
return $percent;
|
|
}
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub getmtime
|
|
{
|
|
my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
|
|
|
|
return $mtime;
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|
|
|
|
sub check_cache
|
|
{
|
|
my $updsource="UPDCACHE";
|
|
my $updfile='';
|
|
my $cacheurl='';
|
|
my $vendorid='';
|
|
my $uuid='';
|
|
my @http_header=();
|
|
my $remote_size=0;
|
|
my $remote_mtime=0;
|
|
my $login='';
|
|
my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
|
|
|
|
my $sourceurl=$_[0];
|
|
my $cfmirror=$_[4];
|
|
|
|
$sourceurl =~ s@\%2b@+@ig;
|
|
$sourceurl =~ s@\%2f@/@ig;
|
|
$sourceurl =~ s@\%7e@~@ig;
|
|
$updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
|
|
$updfile =~ s@\%20@ @ig;
|
|
|
|
if ($cfmirror)
|
|
{
|
|
$uuid = `echo $updfile | md5sum`;
|
|
} else {
|
|
$uuid = `echo $sourceurl | md5sum`;
|
|
}
|
|
|
|
$uuid =~ s/[^0-9a-f]//g;
|
|
$uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
|
|
|
|
$vendorid = $_[3];
|
|
$vendorid =~ tr/A-Z/a-z/;
|
|
|
|
&debuglog("Processing URL $sourceurl");
|
|
&debuglog("Vendor ID is $vendorid");
|
|
&debuglog("UUID is $uuid");
|
|
|
|
if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
|
|
{
|
|
$login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
|
|
if ($proxysettings{'UPSTREAM_PASSWORD'})
|
|
{
|
|
$login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
|
|
}
|
|
}
|
|
|
|
if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
|
|
|
|
$ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
|
|
@http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
|
|
$ENV{'http_proxy'} = '';
|
|
|
|
foreach (@http_header)
|
|
{
|
|
chomp;
|
|
if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
|
|
if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
|
|
}
|
|
|
|
if (
|
|
(-e "$updcachedir/$vendorid/$uuid/$updfile") &&
|
|
($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
|
|
($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
|
|
)
|
|
{
|
|
&debuglog("File exists in cache and is up to date");
|
|
&debuglog("Retrieving file from cache ($updsource) for $hostaddr");
|
|
&setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
|
|
$cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
|
|
}
|
|
elsif (
|
|
($cfmirror == $unique) &&
|
|
($vendorid == "microsoft") &&
|
|
($source_url =~ m@.*[0-9a-f]{40}\.[^\.]+@i)
|
|
)
|
|
{
|
|
# Most Microsoft updates now have an SHA1 hash in the name. These should be treated as unique files.
|
|
# Since it wasn't found in the URL cache, switch to mirror mode and try again using just the filename.
|
|
&debuglog("SHA1: $vendorid $uuid not cached. Reprocessing as mirror $sourceurl");
|
|
$cacheurl = &check_cache($source_url,$hostaddr,$username,$vendorid,$mirror);
|
|
}
|
|
else
|
|
{
|
|
if (-e "$updcachedir/$vendorid/$uuid/$updfile")
|
|
{
|
|
&debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
|
|
&debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
|
|
} else { &debuglog("File not found in cache"); }
|
|
$updsource="DLSOURCE";
|
|
&debuglog("Remote filesize: $remote_size");
|
|
&debuglog("Remote timestamp: $remote_mtime");
|
|
&debuglog("Free disk space: " . &diskfree($updcachedir));
|
|
&debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
|
|
if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
|
|
&debuglog("Retrieving file from source ($updsource) for $hostaddr");
|
|
if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
|
|
{
|
|
&debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
|
|
system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
|
|
}
|
|
$cacheurl=$sourceurl;
|
|
}
|
|
|
|
&writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
|
|
|
|
return $cacheurl;
|
|
}
|
|
|
|
# -------------------------------------------------------------------
|