Subversion Repositories general

Compare Revisions

Ignore whitespace Rev 1150 → Rev 1151

/fsbackup/trunk/fsbackup.pl
11,7 → 11,7
# -c - clean - очиска хранилища с инкрементальным бэкапом и создание нового бэкапа.
 
#############################################
use constant DB_DEF_CACHE_SIZE => 4096000; # Размер кэша для размежения хэша в памяти
use constant DB_DEF_CACHE_SIZE => 40960000; # Размер кэша для размежения хэша в памяти
 
use POSIX;
use File::Find;
23,6 → 23,7
use constant VERB_ERROR => 1; # Output all errors and warnings.
use constant VERB_ALL => 2; # Output all the available data.
 
my $version = "2.0";
my $list_lines_cnt = 0;
my $del_lines_cnt = 0;
my $cur_time = time();
74,9 → 75,11
die "Usage: fsbackup.pl [-n|-f|-h|-c] config_name\n";
}
 
 
require "$config";
 
$cfg_move_old_backup=1 if(!defined($cfg_move_old_backup));
$cfg_exit_on_empty=1 if(!defined($cfg_exit_on_empty));
 
if ( ! -d $cfg_cache_dir){
die "\$cfg_cache_dir ($cfg_cache_dir) not found. Set \$cfg_cache_dir varisble in fsbackup.pl\n";
}
85,11 → 88,7
$cfg_size_limit *= 1024; # Килобайты в байты.
$cfg_maximum_archive_size *= 1024; # Килобайты в байты.
 
if (-d $cfg_root_path){
chdir($cfg_root_path);
} else {
die "Invalid \$cfg_root_path path ($cfg_root_path)\n";
}
chdir($cfg_root_path);
 
if ($ARGV[0] eq "-h"){
$cfg_backup_style = "hash";
117,9 → 116,6
die "Unknown backup_style:\$cfg_backup_style=$cfg_backup_style\n";
}
 
if ($cfg_remote_ftp_mode != 1){
$cfg_remote_ftp_mode = 0;
}
 
if ($cfg_backup_style eq "full_backup" || $cfg_backup_style eq "hash"){
$cfg_new_flag=1;
176,38 → 172,31
 
#----------- Вычисляем уровень инкрементальности.
if ($cfg_increment_level != 0 && $cfg_backup_style eq "backup"){
$cur_increment_level=0;
if(open(INCREMENT_LEVEL, "<$cfg_cache_dir/$cfg_backup_name/.increment_level")) {
$cur_increment_level = <INCREMENT_LEVEL>;
$cur_increment_level++;
close (INCREMENT_LEVEL);
}
else {
print "Can't open increment level file ($cfg_cache_dir/$cfg_backup_name/.increment_level).\n";
$cur_increment_level = 0;
}
 
if ( $cfg_type eq "local"){
opendir( DIR, "$cfg_local_path");
while ($cur_dir = readdir DIR){
if ($cur_dir =~ /\-0\.tar${arc_ext}$/){
$cur_increment_level++;
}
}
closedir (DIR);
if ($cur_increment_level >= $cfg_increment_level){
$cur_increment_level=0;
}
 
} elsif ( $cfg_type eq "remote_ssh"){
 
open (DIR, "$prog_ssh -l $cfg_remote_login $cfg_remote_host 'ls $cfg_remote_path/' |") || print "SSH connection failed: $?\n";
while (<DIR>){
$cur_dir = $_;
if ($cur_dir =~ /\-0\.tar${arc_ext}$/){
$cur_increment_level++;
}
}
close (DIR);
} elsif ( $cfg_type eq "remote_ftp"){
foreach $cur_dir ($ftp->ls()){
if ($cur_dir =~ /\-0\.tar${arc_ext}$/){
$cur_increment_level++;
}
}
}
if ($cur_increment_level >= $cfg_increment_level){
if ($cur_increment_level == 0){
$cfg_new_flag=1;
$cfg_clean_flag=1;
}
if(open(INCREMENT_LEVEL, ">$cfg_cache_dir/$cfg_backup_name/.increment_level")) {
print INCREMENT_LEVEL $cur_increment_level;
close (INCREMENT_LEVEL);
}
else {
print "Can't save increment level to file ($cfg_cache_dir/$cfg_backup_name/.increment_level).\n";
}
print "Current increment number: $cur_increment_level\n" if ($cfg_verbose == &VERB_ALL);
}
################################################
251,14 → 240,15
open (LIST, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list")|| print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list).\n";
flock (LIST, 2);
 
# Список с указанием размеров файлов.
open (LIST_SIZE, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.lsize")|| print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.lsize).\n";
flock (LIST_SIZE, 2);
 
# Создаем список директорий в архиве.
open (DIRS, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.dir")|| print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.dir).\n";
flock (DIRS, 2);
 
open (META, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta")|| print "Can't create meta file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta).\n";
print META "fsbackup $version\n";
print META "increment level: $cur_increment_level\n";
close(META);
 
# Считываем список подлежащих бэкапу директорий в память.
 
while(<DATA>){
289,11 → 279,7
next;
 
} elsif ($cur_path =~ /[\/\w]+/) { # /dir[/file]
if ($cur_path =~ /^$cfg_root_path/){
push @fs_path, "$cur_path";
} else {
push @fs_path, "$cfg_root_path$cur_path";
}
push @fs_path, $cur_path;
 
} else {
print STDERR "Syntax error: $cur_path, ingnored.\n" if ($cfg_verbose >= &VERB_ALL);
307,11 → 293,9
foreach $cur_pathitem (@fs_path){
print "Adding $cur_pathitem....\n" if ($cfg_verbose == &VERB_ALL);
find (\&add_to_backup, $cur_pathitem);
chdir($cfg_root_path); # Переходим в корень, так как find делает chdir.
print "done\n" if ($cfg_verbose == &VERB_ALL);
}
close (LIST);
close (LIST_SIZE);
close (DIRS);
#------------
# Составляем список удаленных файлов.
333,6 → 317,7
untie %active_hash_new;
untie %active_hash_last;
 
chdir ("/"); # Переходим в корень, так как все пути у нас без корневого /
 
# Активируем FTP соединение второй раз.
ftp_connect();
356,7 → 341,8
#------------
# Архивируем и передаем в хранилище.
 
if ($list_lines_cnt == 0 && $del_lines_cnt == 0){
if ($cfg_exit_on_empty == 1 && $list_lines_cnt == 0 && $del_lines_cnt == 0){
print "$cfg_exit_on_empty\n";
print "WARNING: Nothing to backup.\n" if ($cfg_verbose >= &VERB_ALL);
exit;
}
365,8 → 351,8
print "Storing local backup...\n" if ($cfg_verbose == &VERB_ALL);
if ($cfg_backup_style eq "sync"){
if ($cfg_clean_flag == 1){ # Удалить старые копии
print "WARNING: If you really shure to delete $cfg_local_path before sync operatioun uncomment line 'system( \"find \$cfg_local_path ! -path '\$cfg_local_path' -maxdepth 1 -exec \$prog_rm -rf \{\} \\;\");'" if ($cfg_verbose >= &VERB_ALL);
# system( "find $cfg_local_path ! -path '$cfg_local_path' -maxdepth 1 -exec $prog_rm -rf \{\} \\;");
print "WARNING: If you really shure to delete $cfg_local_path before sync operatioun uncomment line 'system( \"find \$cfg_local_path -not -path '\$cfg_local_path' -maxdepth 1 -exec \$prog_rm -rf \{\} \\;\");'" if ($cfg_verbose >= &VERB_ALL);
# system( "find $cfg_local_path -not -path '$cfg_local_path' -maxdepth 1 -exec $prog_rm -rf \{\} \\;");
}
 
system( "cd $cfg_local_path; sh $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del");
378,7 → 364,7
if ($cfg_clean_flag == 1){ # Удалить старые копии
if ($cfg_save_old_backup == 0){
system( "$prog_rm -f $cfg_local_path/*");
} else {
} elsif($cfg_move_old_backup == 1) {
if (! -d "$cfg_local_path/OLD"){
system( "mkdir $cfg_local_path/OLD");
}
387,14 → 373,14
# system( "$prog_rm -f $cfg_local_path/*");
}
}
system( "cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta $cfg_local_path/$backup_file_base.meta") == 0 || print "Local FS .meta copy failed: $?\n";
system( "cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list $cfg_local_path/$backup_file_base.list") == 0 || print "Local FS .list copy failed: $?\n";
system( "cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.lsize $cfg_local_path/$backup_file_base.lsize") == 0 || print "Local FS .lsize copy failed: $?\n";
system( "cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.dir $cfg_local_path/$backup_file_base.dir") == 0 || print "Local FS .dir copy failed: $?\n";
system( "cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del $cfg_local_path/$backup_file_base.del") == 0 || print "Local FS .del copy failed: $?\n";
system( "cp -f $cfg_cache_dir/$cfg_backup_name/.hash $cfg_local_path/$backup_file_base.hash") == 0 || print "Local FS .hash copy failed: $?\n";
#system( "cp -f $cfg_cache_dir/$cfg_backup_name/.hash $cfg_local_path/$backup_file_base.hash") == 0 || print "Local FS .hash copy failed: $?\n";
# Обрабатываем разбиение на тома
for ($arc_block_level=0; $arc_block_level <= $#volume_position; $arc_block_level++){
my $tmp_list_file = crate_tmp_list($arc_block_level, $volume_position[$arc_block_level], $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
my $tmp_list_file = create_tmp_list($arc_block_level, $volume_position[$arc_block_level], $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
system( "$prog_tar -c -f - -T $tmp_list_file $prog_gzip_filter $prog_pgp_filter > $cfg_local_path/$backup_file_base-$arc_block_level.tar${arc_ext}") == 0 || print "Local FS tar backup failed: $?\n";
}
}
403,7 → 389,7
print "Storing remote ssh backup...\n" if ($cfg_verbose == &VERB_ALL);
if ($cfg_backup_style eq "sync"){
if ($cfg_clean_flag == 1){ # Удалить старые копии
system( "$prog_ssh -l $cfg_remote_login $cfg_remote_host find $cfg_remote_path ! -path '$cfg_remote_path' -maxdepth 1 -exec rm -rf \{\} \\;");
system( "$prog_ssh -l $cfg_remote_login $cfg_remote_host find $cfg_remote_path -not -path '$cfg_remote_path' -maxdepth 1 -exec rm -rf \{\} \\;");
}
system( "cat $cfg_cache_dir/$cfg_backup_name/.hash | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/.hash'") == 0 || print "SSH connection failed (store .hash): $?\n";
system( "cat $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/.del'") == 0 || print "SSH connection failed (store .hash): $?\n";
435,7 → 421,6
}
}
system( "cat $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/$backup_file_base.list'") == 0 || print "SSH connection failed (copy .list): $?\n";
system( "cat $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.lsize | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/$backup_file_base.lsize'") == 0 || print "SSH connection failed (copy .lsize): $?\n";
system( "cat $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.dir | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/$backup_file_base.dir'") == 0 || print "SSH connection failed (copy .dir): $?\n";
system( "cat $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/$backup_file_base.del'") == 0 || print "SSH connection failed (copy .del): $?\n";
system( "cat $cfg_cache_dir/$cfg_backup_name/.hash | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/$backup_file_base.hash'") == 0 || print "SSH connection failed (copy .hash): $?\n";
442,7 → 427,7
system( "cat $cfg_cache_dir/$cfg_backup_name/.hash | $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/.hash'") == 0 || print "SSH connection failed (cache .hash): $?\n";
# Обрабатываем разбиение на тома
for ($arc_block_level=0; $arc_block_level <= $#volume_position; $arc_block_level++){
my $tmp_list_file = crate_tmp_list($arc_block_level, $volume_position[$arc_block_level], $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
my $tmp_list_file = create_tmp_list($arc_block_level, $volume_position[$arc_block_level], $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
system( "$prog_tar -c -f - -T $tmp_list_file $prog_gzip_filter $prog_pgp_filter| $prog_ssh -l $cfg_remote_login $cfg_remote_host 'cat - > $cfg_remote_path/$backup_file_base-$arc_block_level.tar${arc_ext}'") == 0 || print "SSH connection failed (tar): $?\n";
}
}
476,8 → 461,6
}
$ftp->delete("$backup_file_base.list");
$ftp->put("$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list", "$backup_file_base.list") || print "Can't PUT .list file to remote FTP server\n";
$ftp->delete("$backup_file_base.lsize");
$ftp->put("$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.lsize", "$backup_file_base.lsize") || print "Can't PUT .lsize file to remote FTP server\n";
$ftp->delete("$backup_file_base.dir");
$ftp->put("$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.dir", "$backup_file_base.dir")|| print "Can't PUT .dir file to remote FTP server\n";
$ftp->delete("$backup_file_base.del");
488,18 → 471,7
$ftp->put("$cfg_cache_dir/$cfg_backup_name/.hash", ".hash")|| print "Can't PUT new .hash file to remote FTP server\n";
# Обрабатываем разбиение на тома
for ($arc_block_level=0; $arc_block_level <= $#volume_position; $arc_block_level++){
# # Проблема была в модуле Net::FTP
# if ($arc_block_level > 0){
# # Переотктываем ftp сессию, для обхода проблемы '2Gb'
# $ftp->quit;
# sleep(5);
# ftp_connect();
# }
 
my $tmp_list_file = crate_tmp_list($arc_block_level, $volume_position[$arc_block_level], $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
$ftp->delete("$backup_file_base-$arc_block_level.list");
$ftp->put("$tmp_list_file", "$backup_file_base-$arc_block_level.list") || print "Can't PUT .list file to remote FTP server\n";
my $tmp_list_file = create_tmp_list($arc_block_level, $volume_position[$arc_block_level], $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
$ftp->delete("$backup_file_base-$arc_block_level.tar${arc_ext}");
open (TAR,"$prog_tar -c -f - -T $tmp_list_file $prog_gzip_filter $prog_pgp_filter|")|| print "tar failed: $?\n";
flock(TAR,1);
525,23 → 497,20
$file_name = $_;
$file_fullpath = $File::Find::name;
$file_dir = $File::Find::dir;
$file_fullpath =~ s/^$cfg_root_path/\.\//;
my $file_fullpath_esc = $file_fullpath;
$file_fullpath_esc =~ s/\'/\'\\\'\'/g;
 
# Создаем список директорий
if ((-d "$cfg_root_path/$file_fullpath") && (! -l "$cfg_root_path/$file_fullpath")){
if ((-d $file_fullpath) && (! -l $file_fullpath)){
if (check_path($file_dir, $file_name) == 1){
if ($cfg_backup_style ne "hash"){
($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $stat_mtime, $stat_time) = stat("$cfg_root_path/$file_fullpath");
($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $stat_mtime, $stat_time) = stat($file_fullpath);
$stat_mode = sprintf ("%04o", $stat_mode & 07777);
$file_fullpath_esc =~ s/^\/(.*)$/$1/;
$stat_uid = getpwuid($stat_uid) || $stat_uid;
$stat_gid = getgrgid($stat_gid) || $stat_gid;
my ($sec,$min,$hour,$mday,$mon,$year) = localtime($stat_time);
$stat_time = sprintf ("%4.4d%2.2d%2.2d%2.2d%2.2d.%2.2d",
$year+1900,$mon+1,$mday,$hour,$min,$sec);
print DIRS "mkdir '$file_fullpath_esc'\n";
print DIRS "mkdir -p '$file_fullpath_esc'\n";
print DIRS "chmod $stat_mode '$file_fullpath_esc'\n";
print DIRS "chown $stat_uid:$stat_gid '$file_fullpath_esc'\n";
print DIRS "touch -t $stat_time '$file_fullpath_esc'\n";
553,6 → 522,8
$cur_backup_size = $stat_size + int(length($file_fullpath)/100.0 + 1)*512 + 1536;
push @volume_position, $list_lines_cnt;
}
$active_hash_new{$file_fullpath} = "D";
check_update($file_fullpath, "D", $file_fullpath, $stat_size);
}
} else {
if ($cfg_stopdir_prune == 1){
561,9 → 532,9
}
}
# Работаем с файлами
} elsif ((-f "$cfg_root_path/$file_fullpath") || (-l "$cfg_root_path/$file_fullpath")){
} elsif ((-f $file_fullpath) || (-l $file_fullpath)){
if (check_path($file_dir, $file_name) == 1){
($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $stat_mtime) = stat("$cfg_root_path/$file_fullpath");
($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $stat_mtime) = stat($file_fullpath);
$checksum_stat= md5_base64("$stat_mtime/$stat_size/$stat_mode/$stat_uid/$stat_gid");
# $file_fullpath_md5 = md5_base64($file_fullpath);
$file_fullpath_md5 = $file_fullpath;
576,8 → 547,8
next;
}
 
if (($cfg_checksum eq "md5") && (! -l "$cfg_root_path/$file_fullpath")){
($md5_checksum_stat, $tmp) = split(/\s+/, `$prog_md5sum '$cfg_root_path/$file_fullpath_esc'`);
if (($cfg_checksum eq "md5") && (! -l $file_fullpath)){
($md5_checksum_stat, $tmp) = split(/\s+/, `$prog_md5sum '$file_fullpath_esc'`);
$active_hash_new{$file_fullpath_md5} = "$checksum_stat/$md5_checksum_stat";
check_update($file_fullpath, "$checksum_stat/$md5_checksum_stat", $file_fullpath_md5, $stat_size);
} else {
593,11 → 564,10
sub check_update{
my ($file, $checksum, $filesum, $stat_size) = @_;
if ( $active_hash_last{$filesum} ne $checksum){
if (($active_hash_last{$filesum} ne $checksum) && ($checksum ne "D")){
if ($cfg_backup_style ne "hash"){
$file =~ s/^\/(.*)$/$1/;
print LIST "$file\n";
print LIST_SIZE "$stat_size\t$file\n";
# Обрабатываем случай разбиения гиганских архивов.
if (-l "/$file"){
616,10 → 586,18
}
$list_lines_cnt++;
}
delete $active_hash_last{$filesum};
if (defined $dbobj_last){
$dbobj_last->del($filesum);
if(($active_hash_last{$filesum} eq "D") && ($checksum ne "D")
|| ($active_hash_last{$filesum} ne "D") && ($checksum eq "D"))
{
# if old entry was a directory and now it's file or link or vice versa, leave it in hash
# to add it later to the delete list
}
else {
delete $active_hash_last{$filesum};
if (defined $dbobj_last){
$dbobj_last->del($filesum);
}
}
}
 
###############################################
632,6 → 610,8
$path = "$dir_name/$file_name";
 
 
 
foreach $item (@fs_notmask){
if ($path =~ /$item/){
return 0;
683,7 → 663,7
 
sub ftp_connect{
if ( $cfg_type eq "remote_ftp"){
$ftp = Net::FTP->new($cfg_remote_host, Timeout => 30, Debug => 0, Passive => $cfg_remote_ftp_mode) || die "Can't connect to ftp server.\n";
$ftp = Net::FTP->new($cfg_remote_host, Timeout => 30, Debug => 0) || die "Can't connect to ftp server.\n";
$ftp->login($cfg_remote_login, $cfg_remote_password) || die "Can't login to ftp server.\n";
$ftp->cwd($cfg_remote_path) || die "Path $cfg_remote_path not found on ftp server.\n";
$ftp->binary();
692,7 → 672,7
###############################################
# Содание списка файлов для помещения в определенный том многотомного архива.
 
sub crate_tmp_list{
sub create_tmp_list{
my ($arc_block_level, $position1, $position2, $full_list_path) = @_;
my ($tmp_list_path, $pos_counter);
 
769,7 → 749,7
 
Backup planner running from C<crontab>. For example:
 
18 4 * * * /usr/local/fsbackup/create_backup.sh
18 4 * * * /root/backup/fsbackup/create_backup.sh
 
=item C<install.pl>
 
809,7 → 789,7
 
Name of backup, single word.
 
=item B<$cfg_cache_dir> = '/usr/local/fsbackup/cache'
=item B<$cfg_cache_dir> = '/root/backup/fsbackup/cache'
 
Path of internal cache directory for local backup method.
 
874,10 → 854,6
 
Password of remote login for remote_ftp storage type.
 
=item B<$cfg_remote_ftp_mode> = 0
 
FTP transfer mode. 0 - Active mode, 1 - Passive mode.
 
=item B<$cfg_local_path> = '/var/backup/'
 
Path of directory to store backup on local file system for local storage type.