Add tarball support to multipastes

Signed-off-by: Florian Pritz <bluewind@xinu.at>

8 files changed, 262 insertions, 2 deletions

diff --git a/NEWS b/NEWS
index 9b3efcba7..d031b9f3d 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,10 @@
 This file lists major, incompatible or otherwise important changes, you should look at it after every update.
 
+2014-11-02 Multipastes can now be downloaded as tarballs. The tarballs are
+           cached and you have to run `php index.php file cron` to clean them up. Calling
+           it more often than once a day is recommended (the example changed to every ten
+           minutes). Also note that the default maximum tarball size is rather low (50MiB),
+           you might want to increase it. Also make sure the phar.so extension is loaded..
 2014-10-29 The sender for emails now has to be configured (config key is "email_from")
 2014-10-19 Postgresl support
 2014-09-20 All PHP errors are now converted to exceptions and execution
diff --git a/application/config/config.php b/application/config/config.php
index 2191b21df..2748e97c0 100644
--- a/application/config/config.php
+++ b/application/config/config.php
@@ -384,6 +384,13 @@ $config['actions_max_age'] = 60*60*24*5; // 5 days
 // Files smaller than this won't be deleted (even if they are old enough)
 $config['small_upload_size'] = 1024*10; // 10KiB
 
+// Maximum size for multipaste tarballs. 0 disables the feature
+$config['tarball_max_size'] = 1024*1024*50; // 50MiB
+
+// Multipaste tarballs older than this will be deleted by the cron job
+// Changing this is not recommended
+$config['tarball_cache_time'] = 60*5; // 5 minutes
+
 
 // Possible values:
 // - apc: needs the apc module and is only useful on long running php processes
diff --git a/application/controllers/file.php b/application/controllers/file.php
index 62cf342b1..292605ff0 100644
--- a/application/controllers/file.php
+++ b/application/controllers/file.php
@@ -108,6 +108,11 @@ class File extends MY_Controller {
 		case "info":
 			return $this->_display_info($id);
 
+		case "tar":
+			if ($is_multipaste) {
+				return $this->_tarball($id);
+			}
+
 		default:
 			if ($is_multipaste) {
 				show_error("Invalid action \"".htmlspecialchars($lexer)."\"");
@@ -370,6 +375,56 @@ class File extends MY_Controller {
 		}
 	}
 
+	private function _tarball($id)
+	{
+		if ($this->mmultipaste->id_exists($id)) {
+			$seen = array();
+			$path = $this->mmultipaste->get_tarball_path($id);
+			$archive = new \service\storage($path);
+
+			if (!$archive->exists()) {
+				$files = $this->mmultipaste->get_files($id);
+
+				$total_size = 0;
+				foreach ($files as $filedata) {
+					$total_size += $filedata["filesize"];
+				}
+
+				if ($total_size > $this->config->item("tarball_max_size")) {
+					show_error("Tarball too large, refusing to create.");
+				}
+
+				$tmpfile = $archive->begin();
+				// create empty tar archive so PharData has something to open
+				file_put_contents($tmpfile, str_repeat("\0", 1024*10));
+				$a = new PharData($tmpfile);
+
+				foreach ($files as $filedata) {
+					$filename = $filedata["filename"];
+					if (isset($seen[$filename]) && $seen[$filename]) {
+						$filename = $filedata["id"]."-".$filedata["filename"];
+					}
+					assert(!isset($seen[$filename]));
+					$a->addFile($this->mfile->file($filedata["hash"]), $filename);
+					$seen[$filename] = true;
+				}
+				$archive->gzip_compress();
+				$archive->commit();
+			}
+
+			// update mtime so the cronjob will keep the file for longer
+			$lock = fopen($archive->get_file(), "r+");
+			flock($lock, LOCK_SH);
+			touch($archive->get_file());
+			flock($lock, LOCK_UN);
+
+			assert(filesize($archive->get_file()) > 0);
+
+			$this->load->driver("ddownload");
+			$this->ddownload->serveFile($archive->get_file(), "$id.tar.gz", "application/x-gzip");
+		}
+	}
+
 	function _non_existent()
 	{
 		$this->data["title"] .= " - Not Found";
@@ -997,6 +1052,24 @@ class File extends MY_Controller {
 	{
 		if (!$this->input->is_cli_request()) return;
 
+		$tarball_dir = $this->config->item("upload_path")."/special/multipaste-tarballs";
+		if (is_dir($tarball_dir)) {
+			$tarball_cache_time = $this->config->item("tarball_cache_time");
+			$it = new RecursiveIteratorIterator(
+				new RecursiveDirectoryIterator($tarball_dir), RecursiveIteratorIterator::SELF_FIRST);
+
+			foreach ($it as $file) {
+				if ($file->isFile()) {
+					if ($file->getMTime() < time() - $tarball_cache_time) {
+						$lock = fopen($file, "r+");
+						flock($lock, LOCK_EX);
+						unlink($file);
+						flock($lock, LOCK_UN);
+					}
+				}
+			}
+		}
+
 		// 0 age disables age checks
 		if ($this->config->item('upload_max_age') == 0) return;
 
@@ -1075,6 +1148,9 @@ class File extends MY_Controller {
 			}
 		}
 		closedir($outer_dh);
+
+		// TODO: clean up special/multipaste-tarballs? cron() already expires
+		// after a rather short time, do we really need this here then?
 	}
 
 	function nuke_id()
diff --git a/application/models/mmultipaste.php b/application/models/mmultipaste.php
index 367e74787..29fc183b6 100644
--- a/application/models/mmultipaste.php
+++ b/application/models/mmultipaste.php
@@ -88,11 +88,20 @@ class Mmultipaste extends CI_Model {
 		return true;
 	}
 
+	public function get_tarball_path($id)
+	{
+		return $this->config->item("upload_path")."/special/multipaste-tarballs/".substr(md5($id), 0, 3)."/$id.tar.gz";
+	}
+
 	public function delete_id($id)
 	{
 		$this->db->where('url_id', $id)
 			->delete('multipaste');
 
+		$path = $this->get_tarball_path($id);
+		$f = new \service\storage($this->config->item("upload_path"), $path);
+		$f->unlink();
+
 		if ($this->id_exists($id))  {
 			return false;
 		}
diff --git a/application/service/storage.php b/application/service/storage.php
new file mode 100644
index 000000000..925827d46
--- /dev/null
+++ b/application/service/storage.php
@@ -0,0 +1,161 @@
+<?php
+/*
+ * Copyright 2014 Florian "Bluewind" Pritz <bluewind@server-speed.net>
+ *
+ * Licensed under AGPLv3
+ * (see COPYING for full license text)
+ *
+ */
+
+namespace service;
+
+/**
+ * This class allows to change a temporary file and replace the original one atomically
+ */
+class storage {
+	private $path;
+	private $tempfile = NULL;
+
+	public function __construct($path)
+	{
+		assert(!is_dir($path));
+
+		$this->path = $path;
+	}
+
+	/**
+	 * Create a temp file which can be written to.
+	 *
+	 * Call commit() once you are done writing.
+	 * Call rollback() to remove the file and throw away any data written.
+	 *
+	 * Calling this multiple times will automatically rollback previous calls.
+	 *
+	 * @return temp file path
+	 */
+	public function begin()
+	{
+		if($this->tempfile !== NULL) {
+			$this->rollback();
+		}
+
+		$this->tempfile = $this->create_tempfile();
+
+		return $this->tempfile;
+	}
+
+	/**
+	 * Create a temporary file. You'll need to remove it yourself when no longer needed.
+	 *
+	 * @return path to the temporary file
+	 */
+	private function create_tempfile()
+	{
+		$dir = dirname($this->get_file());
+		$prefix = basename($this->get_file());
+
+		if (!is_dir($dir)) {
+			mkdir($dir, 0777, true);
+		}
+		assert(is_dir($dir));
+
+		return tempnam($dir, $prefix);
+	}
+
+	/**
+	 * Save the temporary file returned by begin() to the permanent path
+	 * (supplied to the constructor) in an atomic fashion.
+	 */
+	public function commit()
+	{
+		$ret = rename($this->tempfile, $this->get_file());
+		if ($ret) {
+			$this->tempfile = NULL;
+		}
+
+		return $ret;
+	}
+
+	public function exists()
+	{
+		return file_exists($this->get_file());
+	}
+
+	public function get_file()
+	{
+		return $this->path;
+	}
+
+	/**
+	 * Throw away any changes made to the temporary file returned by begin()
+	 */
+	public function rollback()
+	{
+		if ($this->tempfile !== NULL) {
+			unlink($this->tempfile);
+			$this->tempfile = NULL;
+		}
+	}
+
+	public function __destruct()
+	{
+		$this->rollback();
+	}
+
+	/**
+	 * GZIPs the temp file
+	 *
+	 * From http://stackoverflow.com/questions/6073397/how-do-you-create-a-gz-file-using-php
+	 * Based on function by Kioob at:
+	 * http://www.php.net/manual/en/function.gzwrite.php#34955
+	 *
+	 * @param string $source Path to file that should be compressed
+	 * @param integer $level GZIP compression level (default: 6)
+	 * @return boolean true if operation succeeds, false on error
+	 */
+	public function gzip_compress($level = 6){
+		if ($this->tempfile === NULL) {
+			return;
+		}
+
+		$source = $this->tempfile;
+		$file = new storage($source);
+		$dest = $file->begin();
+		$mode = 'wb' . $level;
+		$error = false;
+		$chunk_size = 1024*512;
+
+		if ($fp_out = gzopen($dest, $mode)) {
+			if ($fp_in = fopen($source,'rb')) {
+				while (!feof($fp_in)) {
+					gzwrite($fp_out, fread($fp_in, $chunk_size));
+				}
+				fclose($fp_in);
+			} else {
+				$error = true;
+			}
+			gzclose($fp_out);
+		} else {
+			$error = true;
+		}
+
+		if ($error) {
+			return false;
+		} else {
+			$file->commit();
+			return true;
+		}
+	}
+
+	/**
+	 * Delete the file if it exists.
+	 */
+	public function unlink()
+	{
+		if ($this->exists()) {
+			unlink($this->get_file());
+		}
+	}
+}
+
+# vim: set noet:
diff --git a/application/views/file/upload_form.php b/application/views/file/upload_form.php
index 5051f689b..4434a53cf 100644
--- a/application/views/file/upload_form.php
+++ b/application/views/file/upload_form.php
@@ -97,6 +97,7 @@
 			<dt>/&lt;ID&gt;/</dt><dd>automatically display everything in a sensible way</dd>
 			<dt>/&lt;ID&gt;/qr</dt><dd>display a qr code containing a link to <span class="example">/&lt;ID&gt;/</span></dd>
 			<dt>/&lt;ID&gt;/info</dt><dd>display some information about the multipaste</dd>
+			<dt>/&lt;ID&gt;/tar</dt><dd>download a tarball of all files in the multipaste (files may be renamed to avoid conflicts)</dd>
 		</dl>
 	</div>
 	<div class="col-lg-6">
diff --git a/crontab b/crontab
index 311912df5..d461ba15b 100644
--- a/crontab
+++ b/crontab
@@ -1,2 +1,2 @@
-28 1 * * * php ~/index.php file cron
-28 2 * * * php ~/index.php user cron
\ No newline at end of file
+*/10 * * * * php ~/index.php file cron
+28 * * * * php ~/index.php user cron
diff --git a/install.php b/install.php
index df3868074..8f6530bbd 100644
--- a/install.php
+++ b/install.php
@@ -72,6 +72,7 @@ if ($buf != "0") {
 $mod_groups = array(
 	"thumbnail generation" => array("gd"),
 	"database support" => array("mysql", "mysqli", "pgsql", "pdo_mysql", "pdo_pgsql"),
+	"multipaste tarball support" => array("phar"),
 );
 foreach ($mod_groups as $function => $mods) {
 	$found = 0;