WebSVN – general – /fsbackup/trunk/fsbackup.pl/

#!/usr/bin/perl
# fsbackup - file system backup and synchronization utility. 
#
# http://www.opennet.ru/dev/fsbackup/
# Copyright (c) 2001-2002 by Maxim Chirkov. <mc@tyumen.ru>
# Copyright (c) 2003-2006 by Anatoli Klassen. <anatoli@aksoft.net>
#
# Command line flags:
# -n - create a new (full) archive
# -c - delete old cache and create a full archive
#
# $Id: fsbackup.pl 1258 2006-07-14 12:50:58Z dev $
#
#############################################
use constant DB_DEF_CACHE_SIZE => 40960000; # size of cache in memory

use POSIX;
use File::Find;
use Digest::MD5 qw(md5_base64);
use DB_File;
use BSD::stat;

use constant VERB_SILENT => 0; # Silent mode, suspend all output.
use constant VERB_ERROR  => 1; # Output all errors and warnings.
use constant VERB_ALL    => 2; # Output all the available data.

my $version = "3.0";
my $list_lines_cnt = 0;
my $del_lines_cnt = 0;
my $cur_time = time();
my %active_hash_last;
my %active_hash_new;
my $cfg_new_flag = 0;
my $cfg_clean_flag = 0;
my $config = 0;
my $cur_backup_size = 1536; # tar block size, to calc volumes size
my $backup_file_base;
my $prog_gzip_filter;
my $cur_increment_level;
my $cur_dir;
my $cur_path;
my $cur_file;
my $cur_pathitem;
my $file_fullpath;
my $file_fullpath_md5;
my $key;
my $dbobj_new;
my $dbobj_last;
my $db_hashinfo;
my $db_hashinfo2;
my $file;
my @volume_position=(0);
my @fs_path=();        #  /dir[/file]
my @fs_notpath=();     #  !
my @fs_mask=();        #  =~
my @fs_filemask=();    #  f~
my @fs_dirmask=();     #  d~
my @fs_notmask=();     #  =!
my @fs_notfilemask=(); #  f!
my @fs_notdirmask=();  #  d!

# Load the config
if($ARGV[0] eq "-n" || $ARGV[0] eq "-c") {
  $config = $ARGV[1];
}
else {
  $config = $ARGV[0];
}

if(!-f $config) {
  die "Usage: fsbackup.pl [-n|-c] config_name\n";
}

require "$config";

$cfg_move_old_backup = 1 if(!defined($cfg_move_old_backup));
$cfg_exit_on_empty   = 1 if(!defined($cfg_exit_on_empty));

if(! -d $cfg_cache_dir) {
  die "\$cfg_cache_dir ($cfg_cache_dir) not found. Set \$cfg_cache_dir variable in fsbackup.pl\n";
}

# Convert config values
$cfg_time_limit           *= 60 * 60 * 24;
$cfg_size_limit           *= 1024;
$cfg_maximum_archive_size *= 1024;

# Command line args
if($ARGV[0] eq "-n" || $ARGV[0] eq "-c") {
  $cfg_new_flag = 1;
}

if($ARGV[0] eq "-c") {
  $cfg_clean_flag = 1;
}

# Check config
if($cfg_backup_name !~ /^[\w\d\_.]+$/) {
  die "Found illegal characters in $cfg_backup_name ($cfg_backup_name).";
}

if(!grep {$_ eq $cfg_checksum} ("md5", "timesize")) {
    die "Unknown checksum method:\$cfg_checksum=$cfg_checksum (allowed md5 or timesize)\n";
}

if(! -d $cfg_local_path) {
  die "Can't find \$cfg_local_path ($cfg_local_path)";
}

my($sec, $min, $hour, $mday, $mon, $year) = localtime($cur_time);
$backup_file_base = sprintf("%s-%4.4d.%2.2d.%2.2d.%2.2d.%2.2d.%2.2d",
  $cfg_backup_name, $year+1900, $mon+1, $mday, $hour, $min, $sec);

# Prepare
print "Creating $cfg_backup_name\n" if($cfg_verbose == &VERB_ALL);

chdir($cfg_root_path);

if(! -d "$cfg_cache_dir/$cfg_backup_name") {
  mkdir("$cfg_cache_dir/$cfg_backup_name", 0700);
}

# Calc increment level
if($cfg_increment_level != 0) {
  if(open(INCREMENT_LEVEL, "<$cfg_cache_dir/$cfg_backup_name/.increment_level")) {
    $cur_increment_level = <INCREMENT_LEVEL>;
    $cur_increment_level++;
    close(INCREMENT_LEVEL);
  }
  else {
    print "Can't open increment level file ($cfg_cache_dir/$cfg_backup_name/.increment_level).\n";
    $cur_increment_level = 0;
  }

  if($cur_increment_level >= $cfg_increment_level) {
    $cur_increment_level=0;
  }

  if($cur_increment_level == 0) {
    $cfg_new_flag   = 1;
    $cfg_clean_flag = 1;
  }

  if(open(INCREMENT_LEVEL, ">$cfg_cache_dir/$cfg_backup_name/.increment_level")) {
    print INCREMENT_LEVEL $cur_increment_level;
    close(INCREMENT_LEVEL);
  }
  else {
    print "Can't save increment level to file ($cfg_cache_dir/$cfg_backup_name/.increment_level).\n";
  }
  print "Current increment number: $cur_increment_level\n" if($cfg_verbose == &VERB_ALL);
}

# Load old cache
if(-f "$cfg_cache_dir/$cfg_backup_name/.hash" && $cfg_new_flag == 0) {
  rename("$cfg_cache_dir/$cfg_backup_name/.hash", "$cfg_cache_dir/$cfg_backup_name/.hash.last");
  $db_hashinfo = new DB_File::HASHINFO ;
  $db_hashinfo->{'cachesize'} =  DB_DEF_CACHE_SIZE;
  unless($dbobj_last = tie(%active_hash_last, "DB_File", "$cfg_cache_dir/$cfg_backup_name/.hash.last",
    O_RDWR|O_CREAT, 0644, $db_hashinfo))
  {
    print "WARNING: Error in hash, creating full backup.\n";
    unlink "$cfg_cache_dir/$cfg_backup_name/.hash.last";
  }
}

# Create new cache
unlink("$cfg_cache_dir/$cfg_backup_name/.hash");
$db_hashinfo2 = new DB_File::HASHINFO ;
$db_hashinfo2->{'cachesize'} = 100000;
$dbobj_new = tie(%active_hash_new, "DB_File", "$cfg_cache_dir/$cfg_backup_name/.hash",
  O_RDWR|O_CREAT, 0644, $db_hashinfo2) || print "Can't create or open DB File!\n";

# Save meta info
open(META, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta")
  || print "Can't create meta file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta).\n";
print META "fsbackup $version\n";
print META "increment level: $cur_increment_level\n";
close(META);

# Create list of files to save
open(LIST, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list")
  || print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list).\n";
flock(LIST, 2);

# Load list of dirs and files to backup from the config
while(<DATA>) {
  chomp;
  if(/^\!(.*)$/) {           #  !
    push @fs_notpath, $1;
  }
  elsif(/^\=\~(.*)$/){       #  =~
    push @fs_mask, $1;
  }
  elsif(/^f\~(.*)$/){        #  f~
    push @fs_filemask, $1;
  }
  elsif(/^d\~(.*)$/){        #  d~
    push @fs_dirmask, $1;
  }
  elsif(/^\=\!(.*)$/){       #  =!
    push @fs_notmask, $1;
  }
  elsif(/^f\!(.*)$/){        #  f!
    push @fs_notfilemask, $1;
  }
  elsif(/^d\!(.*)$/){        #  d!
    push @fs_notdirmask, $1;
  }
  elsif(/^#/ || /^\s*$/){    #  comment
    next;
  }
  elsif(/[\/\w]+/) {         #  /dir[/file]
    push @fs_path, $_;
  }
  else {
    print STDERR "Syntax error: $_, ingnored.\n";
  }
}

# Find dirs and files to backup
foreach $cur_pathitem (@fs_path) {
  print "Adding $cur_pathitem ... " if($cfg_verbose == &VERB_ALL);
  find(\&add_to_backup, $cur_pathitem);
  print "done\n" if($cfg_verbose == &VERB_ALL);
}
close(LIST);

# Create list of deleted files and dirs
open(DEL, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del")
  || print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del).\n";
flock (DEL, 2);
while(($file, $key) = each(%active_hash_last)) {
  $file =~ s/\'/\'\\\'\'/g;
  $file =~ s/^\/(.*)$/$1/;
  print DEL "rm -rf '$file'\n";
  $del_lines_cnt++;
}
close(DEL);

# Save the cache
$dbobj_new->sync();
untie %active_hash_new;
untie %active_hash_last;

# Goto root because all paths are without leading /
chdir("/");

# Tar them
if($cfg_exit_on_empty == 1 && $list_lines_cnt == 0 && $del_lines_cnt == 0) {
  print "WARNING: Nothing to backup.\n";
  exit;
}
    
print "Storing local backup\n" if($cfg_verbose == &VERB_ALL);
if($cfg_clean_flag == 1) {        # delete old backup
  if($cfg_save_old_backup == 0) {
    system("$prog_rm -f $cfg_local_path/*");
  }
  elsif($cfg_move_old_backup == 1) {
    system("mkdir $cfg_local_path/OLD") if(!-d "$cfg_local_path/OLD");
    system("$prog_rm -f $cfg_local_path/OLD/*");
    system("mv -f $cfg_local_path/$cfg_backup_name* $cfg_local_path/OLD/");
  }
}
system("cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta $cfg_local_path/$backup_file_base.meta")
  == 0 || print "Local FS .meta copy failed: $?\n";
system("cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list $cfg_local_path/$backup_file_base.list")
  == 0 || print "Local FS .list copy failed: $?\n";
system("cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del $cfg_local_path/$backup_file_base.del")
  == 0 || print "Local FS .del copy failed: $?\n";

# Split to volumes
for($arc_block_level = 0; $arc_block_level <= $#volume_position; $arc_block_level++) {
  my $tmp_list_file = create_tmp_list($arc_block_level, $volume_position[$arc_block_level],
    $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");
  system("$prog_tar -c -f - -n -z -T $tmp_list_file > $cfg_local_path/$backup_file_base-$arc_block_level.tar.gz") == 0 || print "Local FS tar backup failed: $?\n";
}

print "***** Backup successful complete.\n" if($cfg_verbose == &VERB_ALL);
exit (0);

########################################
sub add_to_backup
{
  my($file_name, $file_dir, $md5_checksum_stat, $checksum_stat, $file_fullpath_esc);
  my($tmp, $stat_mode, $stat_uid, $stat_gid, $stat_size, $stat_time, $stat_flags);
 
  $file_name         = $_;
  $file_fullpath     = $File::Find::name;
  $file_dir          = $File::Find::dir;
  $file_fullpath_esc = $file_fullpath;
  $file_fullpath_esc =~ s/\'/\'\\\'\'/g;

  # dir
  if((-d $file_fullpath) && (!-l $file_fullpath)) {
    if(check_path($file_dir, $file_name) == 1) {
      ($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $tmp,
        $stat_time, $tmp, $tmp, $tmp, $tmp, $tmp, $stat_flags, $tmp) = lstat($file_fullpath);
      $checksum_stat     = md5_base64("$stat_time/$stat_mode/$stat_uid/$stat_gid");
      $file_fullpath_esc =~ s/^\/(.*)$/$1/;

      $active_hash_new{$file_fullpath} = "D $checksum_stat";
      check_update($file_fullpath, "D $checksum_stat", $file_fullpath, $stat_size);

      if($stat_flags & UF_NODUMP) {
        push @fs_notpath, "$file_dir/$file_name"; # if nodump - skip all subitems
        if($cfg_stopdir_prune == 1) {
          $File::Find::prune = 1;
          return;
        }
      }
    }
    else {
      if($cfg_stopdir_prune == 1) {
        $File::Find::prune = 1;
        return;
      }
    }
  }
  # file or symlink
  elsif((-f $file_fullpath) || (-l $file_fullpath)) {
    if(check_path($file_dir, $file_name) == 1) {
      ($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $tmp,
        $stat_time, $tmp, $tmp, $tmp, $tmp, $tmp, $stat_flags, $tmp) = lstat($file_fullpath);
      unless($stat_flags & UF_NODUMP) {
        $checksum_stat     = md5_base64("$stat_time/$stat_size/$stat_mode/$stat_uid/$stat_gid");
        $file_fullpath_md5 = $file_fullpath;

        if($cfg_time_limit != 0 && $cur_time - $cfg_time_limit > $stat_time) {
          print "Time limit: $cur_time - $cfg_time_limit > $stat_time, file $file_fullpath ignored.\n"
            if($cfg_verbose == &VERB_ALL);
          next;
        }
        if($cfg_size_limit != 0 && $cfg_size_limit < $stat_size) {
          print "Size limit: $cfg_size_limit < $stat_size, file $file_fullpath ignored.\n"
            if($cfg_verbose == &VERB_ALL);
          next;
        }

        if(($cfg_checksum eq "md5") && (! -l $file_fullpath)) {
          ($md5_checksum_stat, $tmp) = split(/\s+/, `$prog_md5sum '$file_fullpath_esc'`);
          $active_hash_new{$file_fullpath_md5} = "F $checksum_stat/$md5_checksum_stat";
          check_update($file_fullpath, "F $checksum_stat/$md5_checksum_stat", $file_fullpath_md5, $stat_size);
        }
        else {
          $active_hash_new{$file_fullpath} = "F $checksum_stat";
          check_update($file_fullpath, "F $checksum_stat", $file_fullpath, $stat_size);
        }
      }
    }
  }
}

###############################################
# Check if file was updated
sub check_update
{
  my($file, $checksum, $filesum, $stat_size) = @_;
    
  if($active_hash_last{$filesum} ne $checksum) {
    $file =~ s/^\/(.*)$/$1/;
    print LIST "$file\n";
        
    $stat_size = 0 if(-l "/$file");
    
    # next volume
    $cur_backup_size += $stat_size + int(length($file)/100.0 + 1)*512;
    if($cfg_maximum_archive_size > 0 && $cur_backup_size + 10240 >= $cfg_maximum_archive_size) {
      my $old_val = $cur_backup_size - $stat_size - int(length($file)/100.0 + 1)*512;
      my $tmp_pos = $#volume_position+1;
      print "Volume $tmp_pos Done. Size: $old_val\n" if ($cfg_verbose == &VERB_ALL);
      $cur_backup_size = $stat_size + int(length($file)/100.0 + 1)*512 + 1536;
      push @volume_position, $list_lines_cnt;
    }
    $list_lines_cnt++;
  }

  if($active_hash_last{$filesum} && substr($active_hash_last{$filesum}, 0, 1) ne substr($checksum, 0, 1)) {
    # if old entry was a directory and now it's file or link or vice versa, leave it in hash 
    # to add it later to the delete list
  }
  else {
    delete $active_hash_last{$filesum};
    $dbobj_last->del($filesum) if(defined $dbobj_last);
  }
}

###############################################
# 0 - don't add the file
# 1 - add the file
sub check_path
{
  my ($dir_name, $file_name) = @_;
  my ($item, $path);
    
  $path = "$dir_name/$file_name";

  foreach $item (@fs_notmask) {
    return 0 if($path =~ /$item/);
  }

  foreach $item (@fs_notfilemask) {
    return 0 if($file_name =~ /$item/);
  }

  foreach $item (@fs_filemask) {
    return 1 if($file_name =~ /$item/);
  }

  foreach $item (@fs_notdirmask) {
    return 0 if($dir_name =~ /$item/);
  }

  foreach $item (@fs_mask) {
    return 1 if($path =~ /$item/);
  }

  foreach $item (@fs_dirmask) {
    return 1 if($dir_name =~ /$item/);
  }

  foreach $item (@fs_notpath) {
    return 0 if(($dir_name eq $item) || ($path eq $item) || ($dir_name =~ /^$item\//));
  }

  return 1;
}

###############################################
# Split the big list into volume list
sub create_tmp_list
{
  my($arc_block_level, $position1, $position2, $full_list_path) = @_;
  my($tmp_list_path, $pos_counter);

  if($arc_block_level == 0 && $position1 == 0 && $position2 eq '') {
    $tmp_list_path = $full_list_path;
  }
  else {
    $pos_counter = 0;
    $tmp_list_path = "$full_list_path.$arc_block_level";
    open(FULL_LIST, "<$full_list_path") || die "Can't open full list $full_list_path\n";
    flock(FULL_LIST, 1);
    open(TMP_LIST, ">$tmp_list_path") || die "Can't create temp list $tmp_list_path\n";
    flock(TMP_LIST, 2);
    while(<FULL_LIST>) {
      print TMP_LIST $_ if(($pos_counter >= $position1) && ($pos_counter < $position2 || $position2 eq ''));
      $pos_counter++;
    }
    close(TMP_LIST);
    close(FULL_LIST);
  }
  return $tmp_list_path;
}

###############################################
###############################################

__END__

=head1 NAME

fsbackup - file system backup and synchronization utility. 

=head1 SYNOPSIS

    fsbackup.pl [options] <configuration file>

=head1 DESCRIPTION

C<fsbackup.pl> is a incremental backup creation utility. 
C<fsbackup.pl> support backup compression and encryption. Some addition 
scripts allow backups SQL tables from PostgreSQL and MySQL (C<pgsql_backup.sh> 
and C<mysql_backup.sh>)), save system configuration files and list of installed 
packages (C<sysbackup.sh>). 
Backuped with C<fsbackup.pl> files can be recovered by script C<fsrestore.sh>,
backuped with C<sysbackup.sh> system packeges can be reinstalled by C<sysrestore.sh>

=head1 OPTIONS

The following command-line options can be used with C<fsbackup.pl>:

=over

=item C<-n>

Create new backup without checking files in previously stored hash.

=item C<-f>

Create full backup, like as C<-n> option.

=item C<-h>

Only rebuild hash, no storing files in backup archive.

=item C<-c>

Clean incremental backup storage and create new full backup without checking
$cfg_increment_level config parameter.

=head1 ADDITION SCRIPTS

=item C<create_backup.sh>

Backup planner running from C<crontab>. For example: 

18 4 * * * /root/backup/fsbackup/create_backup.sh

=item C<install.pl>

Script to install fsbackup package and some required perl modules.

=item C<fsbackup.pl>

File system backup utility.

=item C<cfg_example>

Example of configuration file.

=item C<scripts/pgsql_backup.sh>

=item C<scripts/mysql_backup.sh>

Script for backup SQL tables from PostreSQL and MySQL.

=item C<scripts/sysbackup.sh>

Script for store system configuration files and information about installed
packages.

=item C<scripts/fsrestore.sh>

Script for restore files backuped by C<fsbackup.pl>.

=item C<scripts/sysrestore.sh>

Script for reinstall packages stored by C<sysbackup.sh>.


=head1 CONFIGURATION FILE

=item B<$cfg_backup_name> = 'test_host'

Name of backup, single word.

=item B<$cfg_cache_dir> = '/root/backup/fsbackup/cache'

Path of internal cache directory for local backup method.

=item B<$prog_md5sum> = 'md5sum -b'

=item B<$prog_tar> = 'tar'

=item B<$prog_rm> = 'rm'

=item B<$cfg_checksum> = 'timesize'

File checksum method: 

timesize - checksum of file attributes (default, best speed) 

md5      - checksum of file attributes + MD5 checksum of file content.

=item B<$cfg_increment_level> = 7

Incremental level (after how many incremental copy make full refresh of backup)

=item B<$cfg_local_path> = '/var/backup/'

Path of directory to store backup on local file system for local storage type.

=item B<$cfg_time_limit> = 0

Limit of file creation time in days. If not 0, don't backup files created or 
modified later then $cfg_time_limit (days).

=item B<$cfg_size_limit> = 0

Limit of maximum file size. If not 0, don't backup files witch size more then 
$cfg_time_limit kilobytes.

=item B<$cfg_root_path> = '/'

Root path for initial chdir.

=item B<$cfg_verbose> = 3

Verbose level.

    0 - Silent mode, suspend all output, except fatal configuration errors.
    1 - Output errors and warnings.
    2 - Output all the  available  data.

=item B<$cfg_save_old_backup> = 1

Save previous backup to OLD directory before rotation or before storing full backup.
    
    0 - don't save old backup
    1 - save old backup.

=item B<$cfg_maximum_archive_size> = 0

Size of maximum size (in KiloBytes) of single unpacked archive file (0 - unlimited file size).

=item B<$cfg_stopdir_prune> = 0

Recursive review of the prohibited directories.
    0 - Recursively to view all contents of directories marked for backup, including contents of directories prohibited by '!', '!d' and '=! rules.
    1 - not use a recursive entrance to directory prohibited for backup (speed is increased, reduces flexibility of customization).

=item B<__DATA__> - list of backuped path and regexp mask.

    /dir[/file] - backup file or directory.
    !/dir[/file] - NOT include this file or directory to backup.
    # - ignore this line.

Mask:

    =~ - regexp mask for include file or directory to backup.
    f~ - regexp file mask for include file to backup.
    d~ - regexp directory mask for include directory to backup.
    =! - regexp mask for NOT include file or directory to backup.
    f! - regexp file mask for NOT include file to backup.
    d! - regexp directory mask for NOT include directory to backup.


Operation priority:

    1. =!
    2. f!
    3. f~
    4. d!
    5. =~
    6. d~
    7. !path
    8. path


=head1 COPYRIGHT

Copyright (c) 2001 by Maxim Chirkov <mc@tyumen.ru>
http://www.opennet.ru/dev/fsbackup/

Copyright (c) 2003-2006 by Anatoli Klassen. <anatoli@aksoft.net>
http://www.26th.net/public/projects/fsbackup/

=head1 BUGS

Look TODO file.

=head1 AUTHORS

Maxim Chirkov <mc@tyumen.ru>
Anatoli Klassen <anatoli@aksoft.net>

=cut
Subversion Repositories general

(root)/fsbackup/trunk/fsbackup.pl/ – Rev 1258
