diff options
| -rw-r--r-- | NEWS | 5 | ||||
| -rw-r--r-- | application/config/config.php | 7 | ||||
| -rw-r--r-- | application/controllers/file.php | 76 | ||||
| -rw-r--r-- | application/models/mmultipaste.php | 9 | ||||
| -rw-r--r-- | application/service/storage.php | 161 | ||||
| -rw-r--r-- | application/views/file/upload_form.php | 1 | ||||
| -rw-r--r-- | crontab | 4 | ||||
| -rw-r--r-- | install.php | 1 |
8 files changed, 262 insertions, 2 deletions
@@ -1,5 +1,10 @@ This file lists major, incompatible or otherwise important changes, you should look at it after every update. +2014-11-02 Multipastes can now be downloaded as tarballs. The tarballs are + cached and you have to run `php index.php file cron` to clean them up. Calling + it more often than once a day is recommended (the example changed to every ten + minutes). Also note that the default maximum tarball size is rather low (50MiB), + you might want to increase it. Also make sure the phar.so extension is loaded.. 2014-10-29 The sender for emails now has to be configured (config key is "email_from") 2014-10-19 Postgresl support 2014-09-20 All PHP errors are now converted to exceptions and execution diff --git a/application/config/config.php b/application/config/config.php index 2191b21df..2748e97c0 100644 --- a/application/config/config.php +++ b/application/config/config.php @@ -384,6 +384,13 @@ $config['actions_max_age'] = 60*60*24*5; // 5 days // Files smaller than this won't be deleted (even if they are old enough) $config['small_upload_size'] = 1024*10; // 10KiB +// Maximum size for multipaste tarballs. 0 disables the feature +$config['tarball_max_size'] = 1024*1024*50; // 50MiB + +// Multipaste tarballs older than this will be deleted by the cron job +// Changing this is not recommended +$config['tarball_cache_time'] = 60*5; // 5 minutes + // Possible values: // - apc: needs the apc module and is only useful on long running php processes diff --git a/application/controllers/file.php b/application/controllers/file.php index 62cf342b1..292605ff0 100644 --- a/application/controllers/file.php +++ b/application/controllers/file.php @@ -108,6 +108,11 @@ class File extends MY_Controller { case "info": return $this->_display_info($id); + case "tar": + if ($is_multipaste) { + return $this->_tarball($id); + } + default: if ($is_multipaste) { show_error("Invalid action \"".htmlspecialchars($lexer)."\""); @@ -370,6 +375,56 @@ class File extends MY_Controller { } } + private function _tarball($id) + { + if ($this->mmultipaste->id_exists($id)) { + $seen = array(); + $path = $this->mmultipaste->get_tarball_path($id); + $archive = new \service\storage($path); + + if (!$archive->exists()) { + $files = $this->mmultipaste->get_files($id); + + $total_size = 0; + foreach ($files as $filedata) { + $total_size += $filedata["filesize"]; + } + + if ($total_size > $this->config->item("tarball_max_size")) { + show_error("Tarball too large, refusing to create."); + } + + $tmpfile = $archive->begin(); + // create empty tar archive so PharData has something to open + file_put_contents($tmpfile, str_repeat("\0", 1024*10)); + $a = new PharData($tmpfile); + + foreach ($files as $filedata) { + $filename = $filedata["filename"]; + if (isset($seen[$filename]) && $seen[$filename]) { + $filename = $filedata["id"]."-".$filedata["filename"]; + } + assert(!isset($seen[$filename])); + $a->addFile($this->mfile->file($filedata["hash"]), $filename); + $seen[$filename] = true; + } + $archive->gzip_compress(); + $archive->commit(); + } + + // update mtime so the cronjob will keep the file for longer + $lock = fopen($archive->get_file(), "r+"); + flock($lock, LOCK_SH); + touch($archive->get_file()); + flock($lock, LOCK_UN); + + assert(filesize($archive->get_file()) > 0); + + $this->load->driver("ddownload"); + $this->ddownload->serveFile($archive->get_file(), "$id.tar.gz", "application/x-gzip"); + } + } + function _non_existent() { $this->data["title"] .= " - Not Found"; @@ -997,6 +1052,24 @@ class File extends MY_Controller { { if (!$this->input->is_cli_request()) return; + $tarball_dir = $this->config->item("upload_path")."/special/multipaste-tarballs"; + if (is_dir($tarball_dir)) { + $tarball_cache_time = $this->config->item("tarball_cache_time"); + $it = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($tarball_dir), RecursiveIteratorIterator::SELF_FIRST); + + foreach ($it as $file) { + if ($file->isFile()) { + if ($file->getMTime() < time() - $tarball_cache_time) { + $lock = fopen($file, "r+"); + flock($lock, LOCK_EX); + unlink($file); + flock($lock, LOCK_UN); + } + } + } + } + // 0 age disables age checks if ($this->config->item('upload_max_age') == 0) return; @@ -1075,6 +1148,9 @@ class File extends MY_Controller { } } closedir($outer_dh); + + // TODO: clean up special/multipaste-tarballs? cron() already expires + // after a rather short time, do we really need this here then? } function nuke_id() diff --git a/application/models/mmultipaste.php b/application/models/mmultipaste.php index 367e74787..29fc183b6 100644 --- a/application/models/mmultipaste.php +++ b/application/models/mmultipaste.php @@ -88,11 +88,20 @@ class Mmultipaste extends CI_Model { return true; } + public function get_tarball_path($id) + { + return $this->config->item("upload_path")."/special/multipaste-tarballs/".substr(md5($id), 0, 3)."/$id.tar.gz"; + } + public function delete_id($id) { $this->db->where('url_id', $id) ->delete('multipaste'); + $path = $this->get_tarball_path($id); + $f = new \service\storage($this->config->item("upload_path"), $path); + $f->unlink(); + if ($this->id_exists($id)) { return false; } diff --git a/application/service/storage.php b/application/service/storage.php new file mode 100644 index 000000000..925827d46 --- /dev/null +++ b/application/service/storage.php @@ -0,0 +1,161 @@ +<?php +/* + * Copyright 2014 Florian "Bluewind" Pritz <bluewind@server-speed.net> + * + * Licensed under AGPLv3 + * (see COPYING for full license text) + * + */ + +namespace service; + +/** + * This class allows to change a temporary file and replace the original one atomically + */ +class storage { + private $path; + private $tempfile = NULL; + + public function __construct($path) + { + assert(!is_dir($path)); + + $this->path = $path; + } + + /** + * Create a temp file which can be written to. + * + * Call commit() once you are done writing. + * Call rollback() to remove the file and throw away any data written. + * + * Calling this multiple times will automatically rollback previous calls. + * + * @return temp file path + */ + public function begin() + { + if($this->tempfile !== NULL) { + $this->rollback(); + } + + $this->tempfile = $this->create_tempfile(); + + return $this->tempfile; + } + + /** + * Create a temporary file. You'll need to remove it yourself when no longer needed. + * + * @return path to the temporary file + */ + private function create_tempfile() + { + $dir = dirname($this->get_file()); + $prefix = basename($this->get_file()); + + if (!is_dir($dir)) { + mkdir($dir, 0777, true); + } + assert(is_dir($dir)); + + return tempnam($dir, $prefix); + } + + /** + * Save the temporary file returned by begin() to the permanent path + * (supplied to the constructor) in an atomic fashion. + */ + public function commit() + { + $ret = rename($this->tempfile, $this->get_file()); + if ($ret) { + $this->tempfile = NULL; + } + + return $ret; + } + + public function exists() + { + return file_exists($this->get_file()); + } + + public function get_file() + { + return $this->path; + } + + /** + * Throw away any changes made to the temporary file returned by begin() + */ + public function rollback() + { + if ($this->tempfile !== NULL) { + unlink($this->tempfile); + $this->tempfile = NULL; + } + } + + public function __destruct() + { + $this->rollback(); + } + + /** + * GZIPs the temp file + * + * From http://stackoverflow.com/questions/6073397/how-do-you-create-a-gz-file-using-php + * Based on function by Kioob at: + * http://www.php.net/manual/en/function.gzwrite.php#34955 + * + * @param string $source Path to file that should be compressed + * @param integer $level GZIP compression level (default: 6) + * @return boolean true if operation succeeds, false on error + */ + public function gzip_compress($level = 6){ + if ($this->tempfile === NULL) { + return; + } + + $source = $this->tempfile; + $file = new storage($source); + $dest = $file->begin(); + $mode = 'wb' . $level; + $error = false; + $chunk_size = 1024*512; + + if ($fp_out = gzopen($dest, $mode)) { + if ($fp_in = fopen($source,'rb')) { + while (!feof($fp_in)) { + gzwrite($fp_out, fread($fp_in, $chunk_size)); + } + fclose($fp_in); + } else { + $error = true; + } + gzclose($fp_out); + } else { + $error = true; + } + + if ($error) { + return false; + } else { + $file->commit(); + return true; + } + } + + /** + * Delete the file if it exists. + */ + public function unlink() + { + if ($this->exists()) { + unlink($this->get_file()); + } + } +} + +# vim: set noet: diff --git a/application/views/file/upload_form.php b/application/views/file/upload_form.php index 5051f689b..4434a53cf 100644 --- a/application/views/file/upload_form.php +++ b/application/views/file/upload_form.php @@ -97,6 +97,7 @@ <dt>/<ID>/</dt><dd>automatically display everything in a sensible way</dd> <dt>/<ID>/qr</dt><dd>display a qr code containing a link to <span class="example">/<ID>/</span></dd> <dt>/<ID>/info</dt><dd>display some information about the multipaste</dd> + <dt>/<ID>/tar</dt><dd>download a tarball of all files in the multipaste (files may be renamed to avoid conflicts)</dd> </dl> </div> <div class="col-lg-6"> @@ -1,2 +1,2 @@ -28 1 * * * php ~/index.php file cron -28 2 * * * php ~/index.php user cron
\ No newline at end of file +*/10 * * * * php ~/index.php file cron +28 * * * * php ~/index.php user cron diff --git a/install.php b/install.php index df3868074..8f6530bbd 100644 --- a/install.php +++ b/install.php @@ -72,6 +72,7 @@ if ($buf != "0") { $mod_groups = array( "thumbnail generation" => array("gd"), "database support" => array("mysql", "mysqli", "pgsql", "pdo_mysql", "pdo_pgsql"), + "multipaste tarball support" => array("phar"), ); foreach ($mod_groups as $function => $mods) { $found = 0; |