summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--NEWS5
-rw-r--r--application/config/config.php7
-rw-r--r--application/controllers/file.php76
-rw-r--r--application/models/mmultipaste.php9
-rw-r--r--application/service/storage.php161
-rw-r--r--application/views/file/upload_form.php1
-rw-r--r--crontab4
-rw-r--r--install.php1
8 files changed, 262 insertions, 2 deletions
diff --git a/NEWS b/NEWS
index 9b3efcba7..d031b9f3d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,10 @@
This file lists major, incompatible or otherwise important changes, you should look at it after every update.
+2014-11-02 Multipastes can now be downloaded as tarballs. The tarballs are
+ cached and you have to run `php index.php file cron` to clean them up. Calling
+ it more often than once a day is recommended (the example changed to every ten
+ minutes). Also note that the default maximum tarball size is rather low (50MiB),
+ you might want to increase it. Also make sure the phar.so extension is loaded..
2014-10-29 The sender for emails now has to be configured (config key is "email_from")
2014-10-19 Postgresl support
2014-09-20 All PHP errors are now converted to exceptions and execution
diff --git a/application/config/config.php b/application/config/config.php
index 2191b21df..2748e97c0 100644
--- a/application/config/config.php
+++ b/application/config/config.php
@@ -384,6 +384,13 @@ $config['actions_max_age'] = 60*60*24*5; // 5 days
// Files smaller than this won't be deleted (even if they are old enough)
$config['small_upload_size'] = 1024*10; // 10KiB
+// Maximum size for multipaste tarballs. 0 disables the feature
+$config['tarball_max_size'] = 1024*1024*50; // 50MiB
+
+// Multipaste tarballs older than this will be deleted by the cron job
+// Changing this is not recommended
+$config['tarball_cache_time'] = 60*5; // 5 minutes
+
// Possible values:
// - apc: needs the apc module and is only useful on long running php processes
diff --git a/application/controllers/file.php b/application/controllers/file.php
index 62cf342b1..292605ff0 100644
--- a/application/controllers/file.php
+++ b/application/controllers/file.php
@@ -108,6 +108,11 @@ class File extends MY_Controller {
case "info":
return $this->_display_info($id);
+ case "tar":
+ if ($is_multipaste) {
+ return $this->_tarball($id);
+ }
+
default:
if ($is_multipaste) {
show_error("Invalid action \"".htmlspecialchars($lexer)."\"");
@@ -370,6 +375,56 @@ class File extends MY_Controller {
}
}
+ private function _tarball($id)
+ {
+ if ($this->mmultipaste->id_exists($id)) {
+ $seen = array();
+ $path = $this->mmultipaste->get_tarball_path($id);
+ $archive = new \service\storage($path);
+
+ if (!$archive->exists()) {
+ $files = $this->mmultipaste->get_files($id);
+
+ $total_size = 0;
+ foreach ($files as $filedata) {
+ $total_size += $filedata["filesize"];
+ }
+
+ if ($total_size > $this->config->item("tarball_max_size")) {
+ show_error("Tarball too large, refusing to create.");
+ }
+
+ $tmpfile = $archive->begin();
+ // create empty tar archive so PharData has something to open
+ file_put_contents($tmpfile, str_repeat("\0", 1024*10));
+ $a = new PharData($tmpfile);
+
+ foreach ($files as $filedata) {
+ $filename = $filedata["filename"];
+ if (isset($seen[$filename]) && $seen[$filename]) {
+ $filename = $filedata["id"]."-".$filedata["filename"];
+ }
+ assert(!isset($seen[$filename]));
+ $a->addFile($this->mfile->file($filedata["hash"]), $filename);
+ $seen[$filename] = true;
+ }
+ $archive->gzip_compress();
+ $archive->commit();
+ }
+
+ // update mtime so the cronjob will keep the file for longer
+ $lock = fopen($archive->get_file(), "r+");
+ flock($lock, LOCK_SH);
+ touch($archive->get_file());
+ flock($lock, LOCK_UN);
+
+ assert(filesize($archive->get_file()) > 0);
+
+ $this->load->driver("ddownload");
+ $this->ddownload->serveFile($archive->get_file(), "$id.tar.gz", "application/x-gzip");
+ }
+ }
+
function _non_existent()
{
$this->data["title"] .= " - Not Found";
@@ -997,6 +1052,24 @@ class File extends MY_Controller {
{
if (!$this->input->is_cli_request()) return;
+ $tarball_dir = $this->config->item("upload_path")."/special/multipaste-tarballs";
+ if (is_dir($tarball_dir)) {
+ $tarball_cache_time = $this->config->item("tarball_cache_time");
+ $it = new RecursiveIteratorIterator(
+ new RecursiveDirectoryIterator($tarball_dir), RecursiveIteratorIterator::SELF_FIRST);
+
+ foreach ($it as $file) {
+ if ($file->isFile()) {
+ if ($file->getMTime() < time() - $tarball_cache_time) {
+ $lock = fopen($file, "r+");
+ flock($lock, LOCK_EX);
+ unlink($file);
+ flock($lock, LOCK_UN);
+ }
+ }
+ }
+ }
+
// 0 age disables age checks
if ($this->config->item('upload_max_age') == 0) return;
@@ -1075,6 +1148,9 @@ class File extends MY_Controller {
}
}
closedir($outer_dh);
+
+ // TODO: clean up special/multipaste-tarballs? cron() already expires
+ // after a rather short time, do we really need this here then?
}
function nuke_id()
diff --git a/application/models/mmultipaste.php b/application/models/mmultipaste.php
index 367e74787..29fc183b6 100644
--- a/application/models/mmultipaste.php
+++ b/application/models/mmultipaste.php
@@ -88,11 +88,20 @@ class Mmultipaste extends CI_Model {
return true;
}
+ public function get_tarball_path($id)
+ {
+ return $this->config->item("upload_path")."/special/multipaste-tarballs/".substr(md5($id), 0, 3)."/$id.tar.gz";
+ }
+
public function delete_id($id)
{
$this->db->where('url_id', $id)
->delete('multipaste');
+ $path = $this->get_tarball_path($id);
+ $f = new \service\storage($this->config->item("upload_path"), $path);
+ $f->unlink();
+
if ($this->id_exists($id)) {
return false;
}
diff --git a/application/service/storage.php b/application/service/storage.php
new file mode 100644
index 000000000..925827d46
--- /dev/null
+++ b/application/service/storage.php
@@ -0,0 +1,161 @@
+<?php
+/*
+ * Copyright 2014 Florian "Bluewind" Pritz <bluewind@server-speed.net>
+ *
+ * Licensed under AGPLv3
+ * (see COPYING for full license text)
+ *
+ */
+
+namespace service;
+
+/**
+ * This class allows to change a temporary file and replace the original one atomically
+ */
+class storage {
+ private $path;
+ private $tempfile = NULL;
+
+ public function __construct($path)
+ {
+ assert(!is_dir($path));
+
+ $this->path = $path;
+ }
+
+ /**
+ * Create a temp file which can be written to.
+ *
+ * Call commit() once you are done writing.
+ * Call rollback() to remove the file and throw away any data written.
+ *
+ * Calling this multiple times will automatically rollback previous calls.
+ *
+ * @return temp file path
+ */
+ public function begin()
+ {
+ if($this->tempfile !== NULL) {
+ $this->rollback();
+ }
+
+ $this->tempfile = $this->create_tempfile();
+
+ return $this->tempfile;
+ }
+
+ /**
+ * Create a temporary file. You'll need to remove it yourself when no longer needed.
+ *
+ * @return path to the temporary file
+ */
+ private function create_tempfile()
+ {
+ $dir = dirname($this->get_file());
+ $prefix = basename($this->get_file());
+
+ if (!is_dir($dir)) {
+ mkdir($dir, 0777, true);
+ }
+ assert(is_dir($dir));
+
+ return tempnam($dir, $prefix);
+ }
+
+ /**
+ * Save the temporary file returned by begin() to the permanent path
+ * (supplied to the constructor) in an atomic fashion.
+ */
+ public function commit()
+ {
+ $ret = rename($this->tempfile, $this->get_file());
+ if ($ret) {
+ $this->tempfile = NULL;
+ }
+
+ return $ret;
+ }
+
+ public function exists()
+ {
+ return file_exists($this->get_file());
+ }
+
+ public function get_file()
+ {
+ return $this->path;
+ }
+
+ /**
+ * Throw away any changes made to the temporary file returned by begin()
+ */
+ public function rollback()
+ {
+ if ($this->tempfile !== NULL) {
+ unlink($this->tempfile);
+ $this->tempfile = NULL;
+ }
+ }
+
+ public function __destruct()
+ {
+ $this->rollback();
+ }
+
+ /**
+ * GZIPs the temp file
+ *
+ * From http://stackoverflow.com/questions/6073397/how-do-you-create-a-gz-file-using-php
+ * Based on function by Kioob at:
+ * http://www.php.net/manual/en/function.gzwrite.php#34955
+ *
+ * @param string $source Path to file that should be compressed
+ * @param integer $level GZIP compression level (default: 6)
+ * @return boolean true if operation succeeds, false on error
+ */
+ public function gzip_compress($level = 6){
+ if ($this->tempfile === NULL) {
+ return;
+ }
+
+ $source = $this->tempfile;
+ $file = new storage($source);
+ $dest = $file->begin();
+ $mode = 'wb' . $level;
+ $error = false;
+ $chunk_size = 1024*512;
+
+ if ($fp_out = gzopen($dest, $mode)) {
+ if ($fp_in = fopen($source,'rb')) {
+ while (!feof($fp_in)) {
+ gzwrite($fp_out, fread($fp_in, $chunk_size));
+ }
+ fclose($fp_in);
+ } else {
+ $error = true;
+ }
+ gzclose($fp_out);
+ } else {
+ $error = true;
+ }
+
+ if ($error) {
+ return false;
+ } else {
+ $file->commit();
+ return true;
+ }
+ }
+
+ /**
+ * Delete the file if it exists.
+ */
+ public function unlink()
+ {
+ if ($this->exists()) {
+ unlink($this->get_file());
+ }
+ }
+}
+
+# vim: set noet:
diff --git a/application/views/file/upload_form.php b/application/views/file/upload_form.php
index 5051f689b..4434a53cf 100644
--- a/application/views/file/upload_form.php
+++ b/application/views/file/upload_form.php
@@ -97,6 +97,7 @@
<dt>/&lt;ID&gt;/</dt><dd>automatically display everything in a sensible way</dd>
<dt>/&lt;ID&gt;/qr</dt><dd>display a qr code containing a link to <span class="example">/&lt;ID&gt;/</span></dd>
<dt>/&lt;ID&gt;/info</dt><dd>display some information about the multipaste</dd>
+ <dt>/&lt;ID&gt;/tar</dt><dd>download a tarball of all files in the multipaste (files may be renamed to avoid conflicts)</dd>
</dl>
</div>
<div class="col-lg-6">
diff --git a/crontab b/crontab
index 311912df5..d461ba15b 100644
--- a/crontab
+++ b/crontab
@@ -1,2 +1,2 @@
-28 1 * * * php ~/index.php file cron
-28 2 * * * php ~/index.php user cron \ No newline at end of file
+*/10 * * * * php ~/index.php file cron
+28 * * * * php ~/index.php user cron
diff --git a/install.php b/install.php
index df3868074..8f6530bbd 100644
--- a/install.php
+++ b/install.php
@@ -72,6 +72,7 @@ if ($buf != "0") {
$mod_groups = array(
"thumbnail generation" => array("gd"),
"database support" => array("mysql", "mysqli", "pgsql", "pdo_mysql", "pdo_pgsql"),
+ "multipaste tarball support" => array("phar"),
);
foreach ($mod_groups as $function => $mods) {
$found = 0;