Subversion Repositories general

#!/usr/bin/perl

# fsbackup - file system backup and synchronization utility. 

#

# http://www.opennet.ru/dev/fsbackup/

# Copyright (c) 2001-2002 by Maxim Chirkov. <mc@tyumen.ru>

# Copyright (c) 2003-2006 by Anatoli Klassen. <anatoli@aksoft.net>

#

# Command line flags:

# -n - create a new (full) archive

# -c - delete old cache and create a full archive

#

# $Id: fsbackup.pl 1258 2006-07-14 12:50:58Z dev $

#

#############################################

use constant DB_DEF_CACHE_SIZE => 40960000; # size of cache in memory

use POSIX;

use File::Find;

use Digest::MD5 qw(md5_base64);

use DB_File;

use BSD::stat;

use constant VERB_SILENT => 0; # Silent mode, suspend all output.

use constant VERB_ERROR  => 1; # Output all errors and warnings.

use constant VERB_ALL    => 2; # Output all the available data.

my $version = "3.0";

my $list_lines_cnt = 0;

my $del_lines_cnt = 0;

my $cur_time = time();

my %active_hash_last;

my %active_hash_new;

my $cfg_new_flag = 0;

my $cfg_clean_flag = 0;

my $config = 0;

my $cur_backup_size = 1536; # tar block size, to calc volumes size

my $backup_file_base;

my $prog_gzip_filter;

my $cur_increment_level;

my $cur_dir;

my $cur_path;

my $cur_file;

my $cur_pathitem;

my $file_fullpath;

my $file_fullpath_md5;

my $key;

my $dbobj_new;

my $dbobj_last;

my $db_hashinfo;

my $db_hashinfo2;

my $file;

my @volume_position=(0);

my @fs_path=();        #  /dir[/file]

my @fs_notpath=();     #  !

my @fs_mask=();        #  =~

my @fs_filemask=();    #  f~

my @fs_dirmask=();     #  d~

my @fs_notmask=();     #  =!

my @fs_notfilemask=(); #  f!

my @fs_notdirmask=();  #  d!

# Load the config

if($ARGV[0] eq "-n" || $ARGV[0] eq "-c") {

  $config = $ARGV[1];

}

else {

  $config = $ARGV[0];

}

if(!-f $config) {

  die "Usage: fsbackup.pl [-n|-c] config_name\n";

}

require "$config";

$cfg_move_old_backup = 1 if(!defined($cfg_move_old_backup));

$cfg_exit_on_empty   = 1 if(!defined($cfg_exit_on_empty));

if(! -d $cfg_cache_dir) {

  die "\$cfg_cache_dir ($cfg_cache_dir) not found. Set \$cfg_cache_dir variable in fsbackup.pl\n";

}

# Convert config values

$cfg_time_limit           *= 60 * 60 * 24;

$cfg_size_limit           *= 1024;

$cfg_maximum_archive_size *= 1024;

# Command line args

if($ARGV[0] eq "-n" || $ARGV[0] eq "-c") {

  $cfg_new_flag = 1;

}

if($ARGV[0] eq "-c") {

  $cfg_clean_flag = 1;

}

# Check config

if($cfg_backup_name !~ /^[\w\d\_.]+$/) {

  die "Found illegal characters in $cfg_backup_name ($cfg_backup_name).";

}

if(!grep {$_ eq $cfg_checksum} ("md5", "timesize")) {

    die "Unknown checksum method:\$cfg_checksum=$cfg_checksum (allowed md5 or timesize)\n";

}

if(! -d $cfg_local_path) {

  die "Can't find \$cfg_local_path ($cfg_local_path)";

}

my($sec, $min, $hour, $mday, $mon, $year) = localtime($cur_time);

$backup_file_base = sprintf("%s-%4.4d.%2.2d.%2.2d.%2.2d.%2.2d.%2.2d",

  $cfg_backup_name, $year+1900, $mon+1, $mday, $hour, $min, $sec);

# Prepare

print "Creating $cfg_backup_name\n" if($cfg_verbose == &VERB_ALL);

chdir($cfg_root_path);

if(! -d "$cfg_cache_dir/$cfg_backup_name") {

  mkdir("$cfg_cache_dir/$cfg_backup_name", 0700);

}

# Calc increment level

if($cfg_increment_level != 0) {

  if(open(INCREMENT_LEVEL, "<$cfg_cache_dir/$cfg_backup_name/.increment_level")) {

    $cur_increment_level = <INCREMENT_LEVEL>;

    $cur_increment_level++;

    close(INCREMENT_LEVEL);

  }

  else {

    print "Can't open increment level file ($cfg_cache_dir/$cfg_backup_name/.increment_level).\n";

    $cur_increment_level = 0;

  }

  if($cur_increment_level >= $cfg_increment_level) {

    $cur_increment_level=0;

  }

  if($cur_increment_level == 0) {

    $cfg_new_flag   = 1;

    $cfg_clean_flag = 1;

  }

  if(open(INCREMENT_LEVEL, ">$cfg_cache_dir/$cfg_backup_name/.increment_level")) {

    print INCREMENT_LEVEL $cur_increment_level;

    close(INCREMENT_LEVEL);

  }

  else {

    print "Can't save increment level to file ($cfg_cache_dir/$cfg_backup_name/.increment_level).\n";

  }

  print "Current increment number: $cur_increment_level\n" if($cfg_verbose == &VERB_ALL);

}

# Load old cache

if(-f "$cfg_cache_dir/$cfg_backup_name/.hash" && $cfg_new_flag == 0) {

  rename("$cfg_cache_dir/$cfg_backup_name/.hash", "$cfg_cache_dir/$cfg_backup_name/.hash.last");

  $db_hashinfo = new DB_File::HASHINFO ;

  $db_hashinfo->{'cachesize'} =  DB_DEF_CACHE_SIZE;

  unless($dbobj_last = tie(%active_hash_last, "DB_File", "$cfg_cache_dir/$cfg_backup_name/.hash.last",

    O_RDWR|O_CREAT, 0644, $db_hashinfo))

  {

    print "WARNING: Error in hash, creating full backup.\n";

    unlink "$cfg_cache_dir/$cfg_backup_name/.hash.last";

  }

}

# Create new cache

unlink("$cfg_cache_dir/$cfg_backup_name/.hash");

$db_hashinfo2 = new DB_File::HASHINFO ;

$db_hashinfo2->{'cachesize'} = 100000;

$dbobj_new = tie(%active_hash_new, "DB_File", "$cfg_cache_dir/$cfg_backup_name/.hash",

  O_RDWR|O_CREAT, 0644, $db_hashinfo2) || print "Can't create or open DB File!\n";

# Save meta info

open(META, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta")

  || print "Can't create meta file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta).\n";

print META "fsbackup $version\n";

print META "increment level: $cur_increment_level\n";

close(META);

# Create list of files to save

open(LIST, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list")

  || print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list).\n";

flock(LIST, 2);

# Load list of dirs and files to backup from the config

while(<DATA>) {

  chomp;

  if(/^\!(.*)$/) {           #  !

    push @fs_notpath, $1;

  }

  elsif(/^\=\~(.*)$/){       #  =~

    push @fs_mask, $1;

  }

  elsif(/^f\~(.*)$/){        #  f~

    push @fs_filemask, $1;

  }

  elsif(/^d\~(.*)$/){        #  d~

    push @fs_dirmask, $1;

  }

  elsif(/^\=\!(.*)$/){       #  =!

    push @fs_notmask, $1;

  }

  elsif(/^f\!(.*)$/){        #  f!

    push @fs_notfilemask, $1;

  }

  elsif(/^d\!(.*)$/){        #  d!

    push @fs_notdirmask, $1;

  }

  elsif(/^#/ || /^\s*$/){    #  comment

    next;

  }

  elsif(/[\/\w]+/) {         #  /dir[/file]

    push @fs_path, $_;

  }

  else {

    print STDERR "Syntax error: $_, ingnored.\n";

  }

}

# Find dirs and files to backup

foreach $cur_pathitem (@fs_path) {

  print "Adding $cur_pathitem ... " if($cfg_verbose == &VERB_ALL);

  find(\&add_to_backup, $cur_pathitem);

  print "done\n" if($cfg_verbose == &VERB_ALL);

}

close(LIST);

# Create list of deleted files and dirs

open(DEL, ">$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del")

  || print "Can't create list file ($cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del).\n";

flock (DEL, 2);

while(($file, $key) = each(%active_hash_last)) {

  $file =~ s/\'/\'\\\'\'/g;

  $file =~ s/^\/(.*)$/$1/;

  print DEL "rm -rf '$file'\n";

  $del_lines_cnt++;

}

close(DEL);

# Save the cache

$dbobj_new->sync();

untie %active_hash_new;

untie %active_hash_last;

# Goto root because all paths are without leading /

chdir("/");

# Tar them

if($cfg_exit_on_empty == 1 && $list_lines_cnt == 0 && $del_lines_cnt == 0) {

  print "WARNING: Nothing to backup.\n";

  exit;

}

print "Storing local backup\n" if($cfg_verbose == &VERB_ALL);

if($cfg_clean_flag == 1) {        # delete old backup

  if($cfg_save_old_backup == 0) {

    system("$prog_rm -f $cfg_local_path/*");

  }

  elsif($cfg_move_old_backup == 1) {

    system("mkdir $cfg_local_path/OLD") if(!-d "$cfg_local_path/OLD");

    system("$prog_rm -f $cfg_local_path/OLD/*");

    system("mv -f $cfg_local_path/$cfg_backup_name* $cfg_local_path/OLD/");

  }

}

system("cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.meta $cfg_local_path/$backup_file_base.meta")

  == 0 || print "Local FS .meta copy failed: $?\n";

system("cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list $cfg_local_path/$backup_file_base.list")

  == 0 || print "Local FS .list copy failed: $?\n";

system("cp -f $cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.del $cfg_local_path/$backup_file_base.del")

  == 0 || print "Local FS .del copy failed: $?\n";

# Split to volumes

for($arc_block_level = 0; $arc_block_level <= $#volume_position; $arc_block_level++) {

  my $tmp_list_file = create_tmp_list($arc_block_level, $volume_position[$arc_block_level],

    $volume_position[$arc_block_level+1], "$cfg_cache_dir/$cfg_backup_name/$cfg_backup_name.list");

  system("$prog_tar -c -f - -n -z -T $tmp_list_file > $cfg_local_path/$backup_file_base-$arc_block_level.tar.gz") == 0 || print "Local FS tar backup failed: $?\n";

}

print "***** Backup successful complete.\n" if($cfg_verbose == &VERB_ALL);

exit (0);

########################################

sub add_to_backup

{

  my($file_name, $file_dir, $md5_checksum_stat, $checksum_stat, $file_fullpath_esc);

  my($tmp, $stat_mode, $stat_uid, $stat_gid, $stat_size, $stat_time, $stat_flags);

  $file_name         = $_;

  $file_fullpath     = $File::Find::name;

  $file_dir          = $File::Find::dir;

  $file_fullpath_esc = $file_fullpath;

  $file_fullpath_esc =~ s/\'/\'\\\'\'/g;

  # dir

  if((-d $file_fullpath) && (!-l $file_fullpath)) {

    if(check_path($file_dir, $file_name) == 1) {

      ($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $tmp,

        $stat_time, $tmp, $tmp, $tmp, $tmp, $tmp, $stat_flags, $tmp) = lstat($file_fullpath);

      $checksum_stat     = md5_base64("$stat_time/$stat_mode/$stat_uid/$stat_gid");

      $file_fullpath_esc =~ s/^\/(.*)$/$1/;

      $active_hash_new{$file_fullpath} = "D $checksum_stat";

      check_update($file_fullpath, "D $checksum_stat", $file_fullpath, $stat_size);

      if($stat_flags & UF_NODUMP) {

        push @fs_notpath, "$file_dir/$file_name"; # if nodump - skip all subitems

        if($cfg_stopdir_prune == 1) {

          $File::Find::prune = 1;

          return;

        }

      }

    }

    else {

      if($cfg_stopdir_prune == 1) {

        $File::Find::prune = 1;

        return;

      }

    }

  }

  # file or symlink

  elsif((-f $file_fullpath) || (-l $file_fullpath)) {

    if(check_path($file_dir, $file_name) == 1) {

      ($tmp, $tmp, $stat_mode, $tmp, $stat_uid, $stat_gid, $tmp, $stat_size, $tmp, $tmp,

        $stat_time, $tmp, $tmp, $tmp, $tmp, $tmp, $stat_flags, $tmp) = lstat($file_fullpath);

      unless($stat_flags & UF_NODUMP) {

        $checksum_stat     = md5_base64("$stat_time/$stat_size/$stat_mode/$stat_uid/$stat_gid");

        $file_fullpath_md5 = $file_fullpath;

        if($cfg_time_limit != 0 && $cur_time - $cfg_time_limit > $stat_time) {

          print "Time limit: $cur_time - $cfg_time_limit > $stat_time, file $file_fullpath ignored.\n"

            if($cfg_verbose == &VERB_ALL);

          next;

        }

        if($cfg_size_limit != 0 && $cfg_size_limit < $stat_size) {

          print "Size limit: $cfg_size_limit < $stat_size, file $file_fullpath ignored.\n"

            if($cfg_verbose == &VERB_ALL);

          next;

        }

        if(($cfg_checksum eq "md5") && (! -l $file_fullpath)) {

          ($md5_checksum_stat, $tmp) = split(/\s+/, `$prog_md5sum '$file_fullpath_esc'`);

          $active_hash_new{$file_fullpath_md5} = "F $checksum_stat/$md5_checksum_stat";

          check_update($file_fullpath, "F $checksum_stat/$md5_checksum_stat", $file_fullpath_md5, $stat_size);

        }

        else {

          $active_hash_new{$file_fullpath} = "F $checksum_stat";

          check_update($file_fullpath, "F $checksum_stat", $file_fullpath, $stat_size);

        }

      }

    }

  }

}

###############################################

# Check if file was updated

sub check_update

{

  my($file, $checksum, $filesum, $stat_size) = @_;

  if($active_hash_last{$filesum} ne $checksum) {

    $file =~ s/^\/(.*)$/$1/;

    print LIST "$file\n";

    $stat_size = 0 if(-l "/$file");

    # next volume

    $cur_backup_size += $stat_size + int(length($file)/100.0 + 1)*512;

    if($cfg_maximum_archive_size > 0 && $cur_backup_size + 10240 >= $cfg_maximum_archive_size) {

      my $old_val = $cur_backup_size - $stat_size - int(length($file)/100.0 + 1)*512;

      my $tmp_pos = $#volume_position+1;

      print "Volume $tmp_pos Done. Size: $old_val\n" if ($cfg_verbose == &VERB_ALL);

      $cur_backup_size = $stat_size + int(length($file)/100.0 + 1)*512 + 1536;

      push @volume_position, $list_lines_cnt;

    }

    $list_lines_cnt++;

  }

  if($active_hash_last{$filesum} && substr($active_hash_last{$filesum}, 0, 1) ne substr($checksum, 0, 1)) {

    # if old entry was a directory and now it's file or link or vice versa, leave it in hash 

    # to add it later to the delete list

  }

  else {

    delete $active_hash_last{$filesum};

    $dbobj_last->del($filesum) if(defined $dbobj_last);

  }

}

###############################################

# 0 - don't add the file

# 1 - add the file

sub check_path

{

  my ($dir_name, $file_name) = @_;

  my ($item, $path);

  $path = "$dir_name/$file_name";

  foreach $item (@fs_notmask) {

    return 0 if($path =~ /$item/);

  }

  foreach $item (@fs_notfilemask) {

    return 0 if($file_name =~ /$item/);

  }

  foreach $item (@fs_filemask) {

    return 1 if($file_name =~ /$item/);

  }

  foreach $item (@fs_notdirmask) {

    return 0 if($dir_name =~ /$item/);

  }

  foreach $item (@fs_mask) {

    return 1 if($path =~ /$item/);

  }

  foreach $item (@fs_dirmask) {

    return 1 if($dir_name =~ /$item/);

  }

  foreach $item (@fs_notpath) {

    return 0 if(($dir_name eq $item) || ($path eq $item) || ($dir_name =~ /^$item\//));

  }

  return 1;

}

###############################################

# Split the big list into volume list

sub create_tmp_list

{

  my($arc_block_level, $position1, $position2, $full_list_path) = @_;

  my($tmp_list_path, $pos_counter);

  if($arc_block_level == 0 && $position1 == 0 && $position2 eq '') {

    $tmp_list_path = $full_list_path;

  }

  else {

    $pos_counter = 0;

    $tmp_list_path = "$full_list_path.$arc_block_level";

    open(FULL_LIST, "<$full_list_path") || die "Can't open full list $full_list_path\n";

    flock(FULL_LIST, 1);

    open(TMP_LIST, ">$tmp_list_path") || die "Can't create temp list $tmp_list_path\n";

    flock(TMP_LIST, 2);

    while(<FULL_LIST>) {

      print TMP_LIST $_ if(($pos_counter >= $position1) && ($pos_counter < $position2 || $position2 eq ''));

      $pos_counter++;

    }

    close(TMP_LIST);

    close(FULL_LIST);

  }

  return $tmp_list_path;

}

###############################################

###############################################

__END__

=head1 NAME

fsbackup - file system backup and synchronization utility. 

=head1 SYNOPSIS

    fsbackup.pl [options] <configuration file>

=head1 DESCRIPTION

C<fsbackup.pl> is a incremental backup creation utility. 

C<fsbackup.pl> support backup compression and encryption. Some addition 

scripts allow backups SQL tables from PostgreSQL and MySQL (C<pgsql_backup.sh> 

and C<mysql_backup.sh>)), save system configuration files and list of installed 

packages (C<sysbackup.sh>). 

Backuped with C<fsbackup.pl> files can be recovered by script C<fsrestore.sh>,

backuped with C<sysbackup.sh> system packeges can be reinstalled by C<sysrestore.sh>

=head1 OPTIONS

The following command-line options can be used with C<fsbackup.pl>:

=over

=item C<-n>

Create new backup without checking files in previously stored hash.

=item C<-f>

Create full backup, like as C<-n> option.

=item C<-h>

Only rebuild hash, no storing files in backup archive.

=item C<-c>

Clean incremental backup storage and create new full backup without checking

$cfg_increment_level config parameter.

=head1 ADDITION SCRIPTS

=item C<create_backup.sh>

Backup planner running from C<crontab>. For example: 

18 4 * * * /root/backup/fsbackup/create_backup.sh

=item C<install.pl>

Script to install fsbackup package and some required perl modules.

=item C<fsbackup.pl>

File system backup utility.

=item C<cfg_example>

Example of configuration file.

=item C<scripts/pgsql_backup.sh>

=item C<scripts/mysql_backup.sh>

Script for backup SQL tables from PostreSQL and MySQL.

=item C<scripts/sysbackup.sh>

Script for store system configuration files and information about installed

packages.

=item C<scripts/fsrestore.sh>

Script for restore files backuped by C<fsbackup.pl>.

=item C<scripts/sysrestore.sh>

Script for reinstall packages stored by C<sysbackup.sh>.

=head1 CONFIGURATION FILE

=item B<$cfg_backup_name> = 'test_host'

Name of backup, single word.

=item B<$cfg_cache_dir> = '/root/backup/fsbackup/cache'

Path of internal cache directory for local backup method.

=item B<$prog_md5sum> = 'md5sum -b'

=item B<$prog_tar> = 'tar'

=item B<$prog_rm> = 'rm'

=item B<$cfg_checksum> = 'timesize'

File checksum method: 

timesize - checksum of file attributes (default, best speed) 

md5      - checksum of file attributes + MD5 checksum of file content.

=item B<$cfg_increment_level> = 7

Incremental level (after how many incremental copy make full refresh of backup)

=item B<$cfg_local_path> = '/var/backup/'

Path of directory to store backup on local file system for local storage type.

=item B<$cfg_time_limit> = 0

Limit of file creation time in days. If not 0, don't backup files created or 

modified later then $cfg_time_limit (days).

=item B<$cfg_size_limit> = 0

Limit of maximum file size. If not 0, don't backup files witch size more then 

$cfg_time_limit kilobytes.

=item B<$cfg_root_path> = '/'

Root path for initial chdir.

=item B<$cfg_verbose> = 3

Verbose level.

    1 - Output errors and warnings.

    2 - Output all the  available  data.

=item B<$cfg_save_old_backup> = 1

Save previous backup to OLD directory before rotation or before storing full backup.

    1 - save old backup.

=item B<$cfg_maximum_archive_size> = 0

Size of maximum size (in KiloBytes) of single unpacked archive file (0 - unlimited file size).

=item B<$cfg_stopdir_prune> = 0

Recursive review of the prohibited directories.

    1 - not use a recursive entrance to directory prohibited for backup (speed is increased, reduces flexibility of customization).

=item B<__DATA__> - list of backuped path and regexp mask.

    /dir[/file] - backup file or directory.

    !/dir[/file] - NOT include this file or directory to backup.

    # - ignore this line.

Mask:

    =~ - regexp mask for include file or directory to backup.

    f~ - regexp file mask for include file to backup.

    d~ - regexp directory mask for include directory to backup.

    =! - regexp mask for NOT include file or directory to backup.

    f! - regexp file mask for NOT include file to backup.

    d! - regexp directory mask for NOT include directory to backup.

Operation priority:

    1. =!

    2. f!

    3. f~

    4. d!

    5. =~

    6. d~

    7. !path

    8. path

=head1 COPYRIGHT

Copyright (c) 2001 by Maxim Chirkov <mc@tyumen.ru>

http://www.opennet.ru/dev/fsbackup/

Copyright (c) 2003-2006 by Anatoli Klassen. <anatoli@aksoft.net>

http://www.26th.net/public/projects/fsbackup/

=head1 BUGS

Look TODO file.

=head1 AUTHORS

Maxim Chirkov <mc@tyumen.ru>

Anatoli Klassen <anatoli@aksoft.net>

=cut