diff options
| -rw-r--r-- | lib/App/BorgRestore.pm | 14 | ||||
| -rw-r--r-- | lib/App/BorgRestore/DB.pm | 16 | ||||
| -rw-r--r-- | lib/App/BorgRestore/PathTimeTable/DB.pm | 48 | ||||
| -rw-r--r-- | lib/App/BorgRestore/PathTimeTable/Memory.pm | 19 | ||||
| -rw-r--r-- | lib/App/BorgRestore/Settings.pm | 14 |
5 files changed, 101 insertions, 10 deletions
diff --git a/lib/App/BorgRestore.pm b/lib/App/BorgRestore.pm index cad8cb1..caf96e9 100644 --- a/lib/App/BorgRestore.pm +++ b/lib/App/BorgRestore.pm @@ -8,6 +8,7 @@ our $VERSION = "3.1.0"; use App::BorgRestore::Borg; use App::BorgRestore::DB; use App::BorgRestore::Helper; +use App::BorgRestore::PathTimeTable::DB; use App::BorgRestore::PathTimeTable::Memory; use App::BorgRestore::Settings; @@ -442,9 +443,15 @@ method _handle_added_archives($borg_archives) { for my $archive (@$add_archives) { my $start = Time::HiRes::gettimeofday(); - my $lookuptable = App::BorgRestore::PathTimeTable::Memory->new({db => $self->{db}}); + my $lookuptable_class = $App::BorgRestore::Settings::prepare_data_in_memory == 1 ? "Memory" : "DB"; + $log->debugf("Using '%s' class for PathTimeTable", $lookuptable_class); + my $lookuptable = "App::BorgRestore::PathTimeTable::$lookuptable_class"->new({db => $self->{db}}); $log->infof("Adding archive %s", $archive); + $self->{db}->begin_work; + $self->{db}->add_archive_name($archive); + my $archive_id = $self->{db}->get_archive_id($archive); + $lookuptable->set_archive_id($archive_id); $self->{borg}->list_archive($archive, sub { my $line = shift; @@ -461,10 +468,7 @@ method _handle_added_archives($borg_archives) { my $borg_time = Time::HiRes::gettimeofday; - $self->{db}->begin_work; - $self->{db}->add_archive_name($archive); - my $archive_id = $self->{db}->get_archive_id($archive); - $lookuptable->save_nodes($archive_id); + $lookuptable->save_nodes(); $self->{db}->commit; $self->{db}->vacuum; $self->{db}->verify_cache_fill_rate_ok(); diff --git a/lib/App/BorgRestore/DB.pm b/lib/App/BorgRestore/DB.pm index 0c6c057..6af14af 100644 --- a/lib/App/BorgRestore/DB.pm +++ b/lib/App/BorgRestore/DB.pm @@ -156,16 +156,26 @@ method get_archives_for_path($path) { return \@ret; } - -method add_path($archive_id, $path, $time) { +method _insert_path($archive_id, $path, $time) { my $st = $self->{dbh}->prepare_cached('insert or ignore into `files` (`path`, `'.$archive_id.'`) values(?, ?)'); $st->execute($path, $time); +} + +method add_path($archive_id, $path, $time) { + $self->_insert_path($archive_id, $path, $time); - $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ?'); + my $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ?'); $st->execute($time, $path); } +method update_path_if_greater($archive_id, $path, $time) { + $self->_insert_path($archive_id, $path, $time); + + my $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ? and `'.$archive_id.'` < ?'); + $st->execute($time, $path, $time); +} + method begin_work() { $self->{dbh}->begin_work(); } diff --git a/lib/App/BorgRestore/PathTimeTable/DB.pm b/lib/App/BorgRestore/PathTimeTable/DB.pm new file mode 100644 index 0000000..ac9969e --- /dev/null +++ b/lib/App/BorgRestore/PathTimeTable/DB.pm @@ -0,0 +1,48 @@ +package App::BorgRestore::PathTimeTable::DB; +use strict; +use warnings; + +use Function::Parameters; + +=head1 NAME + +App::BorgRestore::PathTimeTable::DB - Directly write new archive data to the database + +=head1 DESCRIPTION + +This is used by L<App::BorgRestore> to add new archive data into the database. +Data is written to the database directly and existing data is updated where necessary. + +=cut + +method new($class: $deps = {}) { + return $class->new_no_defaults($deps); +} + +method new_no_defaults($class: $deps = {}) { + my $self = {}; + bless $self, $class; + $self->{deps} = $deps; + return $self; +} + +method set_archive_id($archive_id) { + $self->{archive_id} = $archive_id; +} + +method add_path($path, $time) { + while ($path =~ m#/#) { + $self->{deps}->{db}->update_path_if_greater($self->{archive_id}, $path, $time); + $path =~ s|/[^/]*$||; + } + $self->{deps}->{db}->update_path_if_greater($self->{archive_id}, $path, $time) unless $path eq "."; +} + + +method save_nodes() { + # do nothing because we already write everything to the DB directly +} + +1; + +__END__ diff --git a/lib/App/BorgRestore/PathTimeTable/Memory.pm b/lib/App/BorgRestore/PathTimeTable/Memory.pm index 7aee5b0..012e937 100644 --- a/lib/App/BorgRestore/PathTimeTable/Memory.pm +++ b/lib/App/BorgRestore/PathTimeTable/Memory.pm @@ -5,6 +5,17 @@ use warnings; use Function::Parameters; +=head1 NAME + +App::BorgRestore::PathTimeTable::Memory - In-Memory preparation of new archive data + +=head1 DESCRIPTION + +This is used by L<App::BorgRestore> to add new archive data into the database. +Data is prepared in memory first and only written to the database once. + +=cut + method new($class: $deps = {}) { return $class->new_no_defaults($deps); } @@ -19,6 +30,10 @@ method new_no_defaults($class: $deps = {}) { return $self; } +method set_archive_id($archive_id) { + $self->{archive_id} = $archive_id; +} + method add_path($path, $time) { my @components = split /\//, $path; @@ -45,8 +60,8 @@ method add_path($path, $time) { } } -method save_nodes($archive_id) { - $self->_save_node($archive_id, undef, $self->{lookuptable}); +method save_nodes() { + $self->_save_node($self->{archive_id}, undef, $self->{lookuptable}); } method _save_node($archive_id, $prefix, $node) { diff --git a/lib/App/BorgRestore/Settings.pm b/lib/App/BorgRestore/Settings.pm index 7bf9664..a221502 100644 --- a/lib/App/BorgRestore/Settings.pm +++ b/lib/App/BorgRestore/Settings.pm @@ -79,6 +79,18 @@ The size of the in-memory cache of sqlite in kibibytes. Increasing this may reduce disk IO and improve performance on certain systems when updating the cache. +=item C<$prepare_data_in_memory> + +Default: 1 + +When new archives are added to the cache, the modification time of each parent +directory for a file's path are updated. If this setting is set to 1, these +updates are done in memory before data is written to the database. If it is set +to 0, any changes are written directly to the database. Many values are updated +multiple time, thus writing directly to the database is slower, but preparing +the data in memory may require a substaintial amount of memory. If you run into +out-of-memory problem try setting this to 0. + =back =head2 Example Configuration @@ -92,6 +104,7 @@ cache. {regex => "^/", replacement => "mnt/snapshots/root/"}, ); $sqlite_cache_size = 2097152; + $prepare_data_in_memory = 1; 1; #ensure positive return value @@ -110,6 +123,7 @@ our @backup_prefixes = ( {regex => "^/", replacement => ""}, ); our $sqlite_cache_size = 102400; +our $prepare_data_in_memory = 1; my @configfiles; if (defined $ENV{XDG_CONFIG_HOME} or defined $ENV{HOME}) { |