summaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorFlorian Pritz <bluewind@xinu.at>2018-09-06 18:19:07 +0200
committerFlorian Pritz <bluewind@xinu.at>2018-09-06 18:19:07 +0200
commitb8d19a541782812a4cd02a408344760d83b7b4e0 (patch)
tree35a76837e3cbb16067438d8d94d2399ee6914224 /lib
parent892db03a49333daf7d808395a0a25e81ec2a76ab (diff)
downloadApp-BorgRestore-b8d19a541782812a4cd02a408344760d83b7b4e0.tar.gz
App-BorgRestore-b8d19a541782812a4cd02a408344760d83b7b4e0.tar.xz
Add direct-to-db adding of paths instead of memory only
Signed-off-by: Florian Pritz <bluewind@xinu.at>
Diffstat (limited to 'lib')
-rw-r--r--lib/App/BorgRestore.pm14
-rw-r--r--lib/App/BorgRestore/DB.pm16
-rw-r--r--lib/App/BorgRestore/PathTimeTable/DB.pm48
-rw-r--r--lib/App/BorgRestore/PathTimeTable/Memory.pm19
-rw-r--r--lib/App/BorgRestore/Settings.pm14
5 files changed, 101 insertions, 10 deletions
diff --git a/lib/App/BorgRestore.pm b/lib/App/BorgRestore.pm
index cad8cb1..caf96e9 100644
--- a/lib/App/BorgRestore.pm
+++ b/lib/App/BorgRestore.pm
@@ -8,6 +8,7 @@ our $VERSION = "3.1.0";
use App::BorgRestore::Borg;
use App::BorgRestore::DB;
use App::BorgRestore::Helper;
+use App::BorgRestore::PathTimeTable::DB;
use App::BorgRestore::PathTimeTable::Memory;
use App::BorgRestore::Settings;
@@ -442,9 +443,15 @@ method _handle_added_archives($borg_archives) {
for my $archive (@$add_archives) {
my $start = Time::HiRes::gettimeofday();
- my $lookuptable = App::BorgRestore::PathTimeTable::Memory->new({db => $self->{db}});
+ my $lookuptable_class = $App::BorgRestore::Settings::prepare_data_in_memory == 1 ? "Memory" : "DB";
+ $log->debugf("Using '%s' class for PathTimeTable", $lookuptable_class);
+ my $lookuptable = "App::BorgRestore::PathTimeTable::$lookuptable_class"->new({db => $self->{db}});
$log->infof("Adding archive %s", $archive);
+ $self->{db}->begin_work;
+ $self->{db}->add_archive_name($archive);
+ my $archive_id = $self->{db}->get_archive_id($archive);
+ $lookuptable->set_archive_id($archive_id);
$self->{borg}->list_archive($archive, sub {
my $line = shift;
@@ -461,10 +468,7 @@ method _handle_added_archives($borg_archives) {
my $borg_time = Time::HiRes::gettimeofday;
- $self->{db}->begin_work;
- $self->{db}->add_archive_name($archive);
- my $archive_id = $self->{db}->get_archive_id($archive);
- $lookuptable->save_nodes($archive_id);
+ $lookuptable->save_nodes();
$self->{db}->commit;
$self->{db}->vacuum;
$self->{db}->verify_cache_fill_rate_ok();
diff --git a/lib/App/BorgRestore/DB.pm b/lib/App/BorgRestore/DB.pm
index 0c6c057..6af14af 100644
--- a/lib/App/BorgRestore/DB.pm
+++ b/lib/App/BorgRestore/DB.pm
@@ -156,16 +156,26 @@ method get_archives_for_path($path) {
return \@ret;
}
-
-method add_path($archive_id, $path, $time) {
+method _insert_path($archive_id, $path, $time) {
my $st = $self->{dbh}->prepare_cached('insert or ignore into `files` (`path`, `'.$archive_id.'`)
values(?, ?)');
$st->execute($path, $time);
+}
+
+method add_path($archive_id, $path, $time) {
+ $self->_insert_path($archive_id, $path, $time);
- $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ?');
+ my $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ?');
$st->execute($time, $path);
}
+method update_path_if_greater($archive_id, $path, $time) {
+ $self->_insert_path($archive_id, $path, $time);
+
+ my $st = $self->{dbh}->prepare_cached('update files set `'.$archive_id.'` = ? where `path` = ? and `'.$archive_id.'` < ?');
+ $st->execute($time, $path, $time);
+}
+
method begin_work() {
$self->{dbh}->begin_work();
}
diff --git a/lib/App/BorgRestore/PathTimeTable/DB.pm b/lib/App/BorgRestore/PathTimeTable/DB.pm
new file mode 100644
index 0000000..ac9969e
--- /dev/null
+++ b/lib/App/BorgRestore/PathTimeTable/DB.pm
@@ -0,0 +1,48 @@
+package App::BorgRestore::PathTimeTable::DB;
+use strict;
+use warnings;
+
+use Function::Parameters;
+
+=head1 NAME
+
+App::BorgRestore::PathTimeTable::DB - Directly write new archive data to the database
+
+=head1 DESCRIPTION
+
+This is used by L<App::BorgRestore> to add new archive data into the database.
+Data is written to the database directly and existing data is updated where necessary.
+
+=cut
+
+method new($class: $deps = {}) {
+ return $class->new_no_defaults($deps);
+}
+
+method new_no_defaults($class: $deps = {}) {
+ my $self = {};
+ bless $self, $class;
+ $self->{deps} = $deps;
+ return $self;
+}
+
+method set_archive_id($archive_id) {
+ $self->{archive_id} = $archive_id;
+}
+
+method add_path($path, $time) {
+ while ($path =~ m#/#) {
+ $self->{deps}->{db}->update_path_if_greater($self->{archive_id}, $path, $time);
+ $path =~ s|/[^/]*$||;
+ }
+ $self->{deps}->{db}->update_path_if_greater($self->{archive_id}, $path, $time) unless $path eq ".";
+}
+
+
+method save_nodes() {
+ # do nothing because we already write everything to the DB directly
+}
+
+1;
+
+__END__
diff --git a/lib/App/BorgRestore/PathTimeTable/Memory.pm b/lib/App/BorgRestore/PathTimeTable/Memory.pm
index 7aee5b0..012e937 100644
--- a/lib/App/BorgRestore/PathTimeTable/Memory.pm
+++ b/lib/App/BorgRestore/PathTimeTable/Memory.pm
@@ -5,6 +5,17 @@ use warnings;
use Function::Parameters;
+=head1 NAME
+
+App::BorgRestore::PathTimeTable::Memory - In-Memory preparation of new archive data
+
+=head1 DESCRIPTION
+
+This is used by L<App::BorgRestore> to add new archive data into the database.
+Data is prepared in memory first and only written to the database once.
+
+=cut
+
method new($class: $deps = {}) {
return $class->new_no_defaults($deps);
}
@@ -19,6 +30,10 @@ method new_no_defaults($class: $deps = {}) {
return $self;
}
+method set_archive_id($archive_id) {
+ $self->{archive_id} = $archive_id;
+}
+
method add_path($path, $time) {
my @components = split /\//, $path;
@@ -45,8 +60,8 @@ method add_path($path, $time) {
}
}
-method save_nodes($archive_id) {
- $self->_save_node($archive_id, undef, $self->{lookuptable});
+method save_nodes() {
+ $self->_save_node($self->{archive_id}, undef, $self->{lookuptable});
}
method _save_node($archive_id, $prefix, $node) {
diff --git a/lib/App/BorgRestore/Settings.pm b/lib/App/BorgRestore/Settings.pm
index 7bf9664..a221502 100644
--- a/lib/App/BorgRestore/Settings.pm
+++ b/lib/App/BorgRestore/Settings.pm
@@ -79,6 +79,18 @@ The size of the in-memory cache of sqlite in kibibytes. Increasing this may
reduce disk IO and improve performance on certain systems when updating the
cache.
+=item C<$prepare_data_in_memory>
+
+Default: 1
+
+When new archives are added to the cache, the modification time of each parent
+directory for a file's path are updated. If this setting is set to 1, these
+updates are done in memory before data is written to the database. If it is set
+to 0, any changes are written directly to the database. Many values are updated
+multiple time, thus writing directly to the database is slower, but preparing
+the data in memory may require a substaintial amount of memory. If you run into
+out-of-memory problem try setting this to 0.
+
=back
=head2 Example Configuration
@@ -92,6 +104,7 @@ cache.
{regex => "^/", replacement => "mnt/snapshots/root/"},
);
$sqlite_cache_size = 2097152;
+ $prepare_data_in_memory = 1;
1; #ensure positive return value
@@ -110,6 +123,7 @@ our @backup_prefixes = (
{regex => "^/", replacement => ""},
);
our $sqlite_cache_size = 102400;
+our $prepare_data_in_memory = 1;
my @configfiles;
if (defined $ENV{XDG_CONFIG_HOME} or defined $ENV{HOME}) {