Add direct-to-db adding of paths instead of memory only

Signed-off-by: Florian Pritz <bluewind@xinu.at>

5 files changed, 101 insertions, 10 deletions

diff --git a/lib/App/BorgRestore.pm b/lib/App/BorgRestore.pm
index cad8cb1..caf96e9 100644
--- a/lib/App/BorgRestore.pm
+++ b/lib/App/BorgRestore.pm
@@ -8,6 +8,7 @@ our $VERSION = "3.1.0";
 use App::BorgRestore::Borg;
 use App::BorgRestore::DB;
 use App::BorgRestore::Helper;
+use App::BorgRestore::PathTimeTable::DB;
 use App::BorgRestore::PathTimeTable::Memory;
 use App::BorgRestore::Settings;
 
@@ -442,9 +443,15 @@ method _handle_added_archives($borg_archives) {
 
 	for my $archive (@$add_archives) {
 		my $start = Time::HiRes::gettimeofday();
-		my $lookuptable = App::BorgRestore::PathTimeTable::Memory->new({db => $self->{db}});
+		my $lookuptable_class = $App::BorgRestore::Settings::prepare_data_in_memory == 1 ? "Memory" : "DB";
+		$log->debugf("Using '%s' class for PathTimeTable", $lookuptable_class);
+		my $lookuptable = "App::BorgRestore::PathTimeTable::$lookuptable_class"->new({db => $self->{db}});
 
 		$log->infof("Adding archive %s", $archive);
+		$self->{db}->begin_work;
+		$self->{db}->add_archive_name($archive);
+		my $archive_id = $self->{db}->get_archive_id($archive);
+		$lookuptable->set_archive_id($archive_id);
 
 		$self->{borg}->list_archive($archive, sub {
 			my $line = shift;
@@ -461,10 +468,7 @@ method _handle_added_archives($borg_archives) {
 
 		my $borg_time = Time::HiRes::gettimeofday;
 
-		$self->{db}->begin_work;
-		$self->{db}->add_archive_name($archive);
-		my $archive_id = $self->{db}->get_archive_id($archive);
-		$lookuptable->save_nodes($archive_id);
+		$lookuptable->save_nodes();
 		$self->{db}->commit;
 		$self->{db}->vacuum;
 		$self->{db}->verify_cache_fill_rate_ok();
diff --git a/lib/App/BorgRestore/DB.pm b/lib/App/BorgRestore/DB.pm
index 0c6c057..6af14af 100644
--- a/lib/App/BorgRestore/DB.pm
+++ b/lib/App/BorgRestore/DB.pm
@@ -156,16 +156,26 @@ method get_archives_for_path($path) {
 	return \@ret;
 }
 
-
-method add_path($archive_id, $path, $time) {
+method _insert_path($archive_id, $path, $time) {
 	my $st = $self->{dbh}->prepare_cached('insert or ignore into `files` (`path`, `'.$archive_id.'`)
 		values(?, ?)');
 	$st->execute($path, $time);
+}
+
+method add_path($archive_id, $path, $time) {
+	$self->_insert_path($archive_id, $path, $time);
 
-	$st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ?');
+	my $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ?');
 	$st->execute($time, $path);
 }
 
+method update_path_if_greater($archive_id, $path, $time) {
+	$self->_insert_path($archive_id, $path, $time);
+
+	my $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ? and `'.$archive_id.'` < ?');
+	$st->execute($time, $path, $time);
+}
+
 method begin_work() {
 	$self->{dbh}->begin_work();
 }
diff --git a/lib/App/BorgRestore/PathTimeTable/DB.pm b/lib/App/BorgRestore/PathTimeTable/DB.pm
new file mode 100644
index 0000000..ac9969e
--- /dev/null
+++ b/lib/App/BorgRestore/PathTimeTable/DB.pm
@@ -0,0 +1,48 @@
+package App::BorgRestore::PathTimeTable::DB;
+use strict;
+use warnings;
+
+use Function::Parameters;
+
+=head1 NAME
+
+App::BorgRestore::PathTimeTable::DB - Directly write new archive data to the database
+
+=head1 DESCRIPTION
+
+This is used by L<App::BorgRestore> to add new archive data into the database.
+Data is written to the database directly and existing data is updated where necessary.
+
+=cut
+
+method new($class: $deps = {}) {
+	return $class->new_no_defaults($deps);
+}
+
+method new_no_defaults($class: $deps = {}) {
+	my $self = {};
+	bless $self, $class;
+	$self->{deps} = $deps;
+	return $self;
+}
+
+method set_archive_id($archive_id) {
+	$self->{archive_id} = $archive_id;
+}
+
+method add_path($path, $time) {
+	while ($path =~ m#/#) {
+		$self->{deps}->{db}->update_path_if_greater($self->{archive_id}, $path, $time);
+		$path =~ s|/[^/]*$||;
+	}
+	$self->{deps}->{db}->update_path_if_greater($self->{archive_id}, $path, $time) unless $path eq ".";
+}
+
+
+method save_nodes() {
+	# do nothing because we already write everything to the DB directly
+}
+
+1;
+
+__END__
diff --git a/lib/App/BorgRestore/PathTimeTable/Memory.pm b/lib/App/BorgRestore/PathTimeTable/Memory.pm
index 7aee5b0..012e937 100644
--- a/lib/App/BorgRestore/PathTimeTable/Memory.pm
+++ b/lib/App/BorgRestore/PathTimeTable/Memory.pm
@@ -5,6 +5,17 @@ use warnings;
 
 use Function::Parameters;
 
+=head1 NAME
+
+App::BorgRestore::PathTimeTable::Memory - In-Memory preparation of new archive data
+
+=head1 DESCRIPTION
+
+This is used by L<App::BorgRestore> to add new archive data into the database.
+Data is prepared in memory first and only written to the database once.
+
+=cut
+
 method new($class: $deps = {}) {
 	return $class->new_no_defaults($deps);
 }
@@ -19,6 +30,10 @@ method new_no_defaults($class: $deps = {}) {
 	return $self;
 }
 
+method set_archive_id($archive_id) {
+	$self->{archive_id} = $archive_id;
+}
+
 method add_path($path, $time) {
 	my @components = split /\//, $path;
 
@@ -45,8 +60,8 @@ method add_path($path, $time) {
 	}
 }
 
-method save_nodes($archive_id) {
-	$self->_save_node($archive_id, undef, $self->{lookuptable});
+method save_nodes() {
+	$self->_save_node($self->{archive_id}, undef, $self->{lookuptable});
 }
 
 method _save_node($archive_id, $prefix, $node) {
diff --git a/lib/App/BorgRestore/Settings.pm b/lib/App/BorgRestore/Settings.pm
index 7bf9664..a221502 100644
--- a/lib/App/BorgRestore/Settings.pm
+++ b/lib/App/BorgRestore/Settings.pm
@@ -79,6 +79,18 @@ The size of the in-memory cache of sqlite in kibibytes. Increasing this may
 reduce disk IO and improve performance on certain systems when updating the
 cache.
 
+=item C<$prepare_data_in_memory>
+
+Default: 1
+
+When new archives are added to the cache, the modification time of each parent
+directory for a file's path are updated. If this setting is set to 1, these
+updates are done in memory before data is written to the database. If it is set
+to 0, any changes are written directly to the database. Many values are updated
+multiple time, thus writing directly to the database is slower, but preparing
+the data in memory may require a substaintial amount of memory. If you run into
+out-of-memory problem try setting this to 0.
+
 =back
 
 =head2 Example Configuration
@@ -92,6 +104,7 @@ cache.
  	{regex => "^/", replacement => "mnt/snapshots/root/"},
  );
  $sqlite_cache_size = 2097152;
+ $prepare_data_in_memory = 1;
 
  1; #ensure positive return value
 
@@ -110,6 +123,7 @@ our @backup_prefixes = (
 	{regex => "^/", replacement => ""},
 );
 our $sqlite_cache_size = 102400;
+our $prepare_data_in_memory = 1;
 my @configfiles;
 
 if (defined $ENV{XDG_CONFIG_HOME} or defined $ENV{HOME}) {