Support multiple files with the same hash

Signed-off-by: Florian Pritz <bluewind@xinu.at>

8 files changed, 231 insertions, 131 deletions

diff --git a/application/config/migration.php b/application/config/migration.php
index 75e9bc19b..952fbf056 100644
--- a/application/config/migration.php
+++ b/application/config/migration.php
@@ -21,7 +21,7 @@ $config['migration_enabled'] = true;
 | be upgraded / downgraded to.
 |
 */
-$config['migration_version'] = 13;
+$config['migration_version'] = 14;
 
 
 /*
diff --git a/application/controllers/file.php b/application/controllers/file.php
index 5e443017c..a653d8e74 100644
--- a/application/controllers/file.php
+++ b/application/controllers/file.php
@@ -118,7 +118,7 @@ class File extends MY_Controller {
 
 		$etag = "";
 		foreach ($files as $filedata) {
-			$etag = sha1($etag.$filedata["hash"]);
+			$etag = sha1($etag.$filedata["data_id"]);
 		}
 
 		// handle some common "lexers" here
@@ -156,7 +156,7 @@ class File extends MY_Controller {
 			handle_etag($etag);
 
 			$filedata = $files[0];
-			$filepath = $this->mfile->file($filedata["hash"]);
+			$filepath = $this->mfile->file($filedata["data_id"]);
 			$this->ddownload->serveFile($filepath, $filedata["filename"], "text/plain");
 			exit();
 		}
@@ -164,7 +164,7 @@ class File extends MY_Controller {
 		$this->load->library("output_cache");
 
 		foreach ($files as $key => $filedata) {
-			$file = $this->mfile->file($filedata['hash']);
+			$file = $this->mfile->file($filedata['data_id']);
 			$pygments = new \libraries\Pygments($file, $filedata["mimetype"], $filedata["filename"]);
 
 			// autodetect the lexer for highlighting if the URL contains a / after the ID (/ID/)
@@ -296,9 +296,9 @@ class File extends MY_Controller {
 	{
 		// highlight the file and cache the result, fall back to plain text if $lexer fails
 		foreach (array($lexer, "text") as $lexer) {
-			$highlit = cache_function($filedata['hash'].'_'.$lexer, 100,
+			$highlit = cache_function($filedata['data_id'].'_'.$lexer, 100,
 									  function() use ($filedata, $lexer, $is_multipaste) {
-				$file = $this->mfile->file($filedata['hash']);
+				$file = $this->mfile->file($filedata['data_id']);
 				if ($lexer == "rmd") {
 					ob_start();
 
@@ -347,7 +347,7 @@ class File extends MY_Controller {
 	private function _tooltip_for_image($filedata)
 	{
 		$filesize = format_bytes($filedata["filesize"]);
-		$file = $this->mfile->file($filedata["hash"]);
+		$file = $this->mfile->file($filedata["data_id"]);
 		$upload_date = date("r", $filedata["date"]);
 
 		$height = 0;
@@ -444,7 +444,7 @@ class File extends MY_Controller {
 						$filename = $filedata["id"]."-".$filedata["filename"];
 					}
 					assert(!isset($seen[$filename]));
-					$a->addFile($this->mfile->file($filedata["hash"]), $filename);
+					$a->addFile($this->mfile->file($filedata["data_id"]), $filename);
 					$seen[$filename] = true;
 				}
 				$archive->gzip_compress();
@@ -501,7 +501,7 @@ class File extends MY_Controller {
 
 				if (count($ids) == 1) {
 					$filedata = $this->mfile->get_filedata($id);
-					$file = $this->mfile->file($filedata['hash']);
+					$file = $this->mfile->file($filedata['data_id']);
 					$pygments = new \libraries\Pygments($file, $filedata["mimetype"], $filedata["filename"]);
 					$lexer = $pygments->should_highlight();
 
@@ -570,9 +570,9 @@ class File extends MY_Controller {
 		if ($repaste_id) {
 			$filedata = $this->mfile->get_filedata($repaste_id);
 
-			$pygments = new \libraries\Pygments($this->mfile->file($filedata["hash"]), $filedata["mimetype"], $filedata["filename"]);
+			$pygments = new \libraries\Pygments($this->mfile->file($filedata["data_id"]), $filedata["mimetype"], $filedata["filename"]);
 			if ($filedata !== false && $pygments->can_highlight()) {
-				$this->data["textarea_content"] = file_get_contents($this->mfile->file($filedata["hash"]));
+				$this->data["textarea_content"] = file_get_contents($this->mfile->file($filedata["data_id"]));
 			}
 		}
 
@@ -609,11 +609,11 @@ class File extends MY_Controller {
 			throw new \exceptions\ApiException("file/thumbnail/filedata-unavailable", "Failed to get file data");
 		}
 
-		$cache_key = $filedata['hash'].'_thumb_'.$thumb_size;
+		$cache_key = $filedata['data_id'].'_thumb_'.$thumb_size;
 
 		$thumb = cache_function($cache_key, 100, function() use ($filedata, $thumb_size){
 			$CI =& get_instance();
-			$img = new libraries\Image($this->mfile->file($filedata["hash"]));
+			$img = new libraries\Image($this->mfile->file($filedata["data_id"]));
 			$img->makeThumb($thumb_size, $thumb_size);
 			$thumb = $img->get(IMAGETYPE_JPEG);
 			return $thumb;
@@ -631,9 +631,11 @@ class File extends MY_Controller {
 
 		$user = $this->muser->get_userid();
 
+		// TODO: move to \service\files and possibly use \s\f::history()
 		$query = $this->db
-			->select('id, filename, mimetype, date, hash, filesize, user')
+			->select('files.id, filename, mimetype, files.date, hash, file_storage.id storage_id, filesize, user')
 			->from('files')
+			->join('file_storage', 'file_storage.id = files.file_storage_id')
 			->where('
 				(user = '.$this->db->escape($user).')
 				AND (
@@ -645,12 +647,14 @@ class File extends MY_Controller {
 
 		foreach($query as $key => $item) {
 			assert($item["user"] === $user);
+			$item["data_id"] = $item['hash']."-".$item['id'];
+			$query[$key]["data_id"] =  $item["data_id"];
 			if (!$this->mfile->valid_id($item["id"])) {
 				unset($query[$key]);
 				continue;
 			}
 			$query[$key]["tooltip"] = $this->_tooltip_for_image($item);
-			$query[$key]["orientation"] = libraries\Image::get_exif_orientation($this->mfile->file($item["hash"]));
+			$query[$key]["orientation"] = libraries\Image::get_exif_orientation($this->mfile->file($item["data_id"]));
 		}
 
 		$this->data["items"] = $query;
@@ -1011,25 +1015,28 @@ class File extends MY_Controller {
 			"sess_expiration" => $this->config->item("sess_expiration"),
 		);
 
-		$query = $this->db->select('hash, id, user, date')
+		$query = $this->db->select('file_storage_id storage_id, id, user, date')
 			->from('files')
 			->where("user", 0)
 			->where("date <", $oldest_session_time)
 			->get()->result_array();
 
 		foreach($query as $row) {
+			$row['data_id'] = $row['hash'].'-'.$row['storage_id'];
 			\service\files::valid_id($row, $config, $this->mfile, time());
 		}
 
 		// 0 age disables age checks
 		if ($this->config->item('upload_max_age') == 0) return;
 
-		$query = $this->db->select('hash, id, user, date')
+		$query = $this->db->select('hash, files.id, user, files.date, file_storage.id storage_id')
 			->from('files')
-			->where('date <', $oldest_time)
+			->join('file_storage', "file_storage.id = files.file_storage_id")
+			->where('files.date <', $oldest_time)
 			->get()->result_array();
 
 		foreach($query as $row) {
+			$row['data_id'] = $row['hash'].'-'.$row['storage_id'];
 			\service\files::valid_id($row, $config, $this->mfile, time());
 		}
 	}
@@ -1056,13 +1063,17 @@ class File extends MY_Controller {
 					continue;
 				}
 
-				$query = $this->db->select('hash')
-					->from('files')
-					->where('hash', $file)
+				list($hash, $storage_id) = explode("-", $file);
+
+				$query = $this->db->select('hash, id')
+					->from('file_storage')
+					->where('hash', $hash)
+					->where('id', $storage_id)
 					->limit(1)
 					->get()->row_array();
 
 				if (empty($query)) {
+					$this->mfile->delete_data_id($file);
 					unlink($upload_path."/".$dir."/".$file);
 				} else {
 					$empty = false;
@@ -1094,9 +1105,9 @@ class File extends MY_Controller {
 			return;
 		}
 
-		$hash = $file_data["hash"];
-		$this->mfile->delete_hash($hash);
-		echo "removed hash \"$hash\"\n";
+		$data_id = $file_data["data_id"];
+		$this->mfile->delete_data_id($data_id);
+		echo "removed data_id \"$data_id\"\n";
 	}
 
 	function update_file_metadata()
@@ -1105,26 +1116,23 @@ class File extends MY_Controller {
 
 		$chunk = 500;
 
-		$total = $this->db->count_all("files");
+		$total = $this->db->count_all("file_storage");
 
 		for ($limit = 0; $limit < $total; $limit += $chunk) {
-			$query = $this->db->select('hash')
-				->from('files')
-				->group_by('hash')
-				->limit($limit, $chunk)
+			$query = $this->db->select('hash, id')
+				->from('file_storage')
+				->limit($chunk, $limit)
 				->get()->result_array();
 
 			foreach ($query as $key => $item) {
-				$hash = $item["hash"];
-				$filesize = intval(filesize($this->mfile->file($hash)));
-				$mimetype = mimetype($this->mfile->file($hash));
+				$data_id = $item["hash"].'-'.$item['id'];
+				$mimetype = mimetype($this->mfile->file($data_id));
 
-				$this->db->where('hash', $hash)
+				$this->db->where('id', $item['id'])
 					->set(array(
-						'filesize' => $filesize,
 						'mimetype' => $mimetype,
 					))
-					->update('files');
+					->update('file_storage');
 			}
 		}
 	}
diff --git a/application/helpers/filebin_helper.php b/application/helpers/filebin_helper.php
index 7adab7279..563364d32 100644
--- a/application/helpers/filebin_helper.php
+++ b/application/helpers/filebin_helper.php
@@ -324,4 +324,31 @@ function mimetype($file) {
 	return $mimetype;
 }
 
+function files_are_equal($a, $b)
+{
+	$chunk_size = 8*1024;
+
+	// Check if filesize is different
+	if (filesize($a) !== filesize($b)) {
+		return false;
+	}
+
+	// Check if content is different
+	$ah = fopen($a, 'rb');
+	$bh = fopen($b, 'rb');
+
+	$result = true;
+	while (!feof($ah) && !feof($bh)) {
+		if (fread($ah, $chunk_size) !== fread($bh, $chunk_size)) {
+			$result = false;
+			break;
+		}
+	}
+
+	fclose($ah);
+	fclose($bh);
+
+	return $result;
+}
+
 # vim: set noet:
diff --git a/application/libraries/Tempfile.php b/application/libraries/Tempfile.php
new file mode 100644
index 000000000..f42efbfdf
--- /dev/null
+++ b/application/libraries/Tempfile.php
@@ -0,0 +1,31 @@
+<?php
+/*
+ * Copyright 2015 Florian "Bluewind" Pritz <bluewind@server-speed.net>
+ *
+ * Licensed under AGPLv3
+ * (see COPYING for full license text)
+ *
+ */
+
+namespace libraries;
+
+class Tempfile {
+	private $file;
+
+	public function __construct()
+	{
+		$this->file = tempnam(sys_get_temp_dir(), "tempfile");
+	}
+
+	public function __destruct()
+	{
+		if (file_exists($this->file)) {
+			unlink($this->file);
+		}
+	}
+
+	public function get_file()
+	{
+		return $this->file;
+	}
+}
diff --git a/application/migrations/014_deduplicate_file_storage.php b/application/migrations/014_deduplicate_file_storage.php
index 8f8f40430..d01ab03a9 100644
--- a/application/migrations/014_deduplicate_file_storage.php
+++ b/application/migrations/014_deduplicate_file_storage.php
@@ -7,41 +7,62 @@ class Migration_deduplicate_file_storage extends CI_Migration {
 	{
 		$prefix = $this->db->dbprefix;
 
-		// FIXME: use prefix
-
 		if ($this->db->dbdriver == 'postgre') {
-			throw new \exceptions\ApiException("migration/postgres/not-implemented", "migration 14 not implemented yet for postgres");
+			throw new \exceptions\ApiException("migration/postgres/not-implemented", "migration 14 not yet implemented for postgres");
 		} else {
 			$this->db->query('
-				CREATE TABLE `file_storage` (
+				CREATE TABLE `'.$prefix.'file_storage` (
 					`id` int(11) NOT NULL AUTO_INCREMENT,
 					`filesize` int(11) NOT NULL,
 					`mimetype` varchar(255) NOT NULL,
 					`hash` char(32) NOT NULL,
-					`hash_collision_counter` int(11) NOT NULL,
+					`date` int(11) NOT NULL,
 					PRIMARY KEY (`id`),
-					UNIQUE KEY `data_id` (`hash`, `hash_collision_counter`)
+					UNIQUE KEY `data_id` (`id`, `hash`)
 				) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 			');
 			$this->db->query('
-				ALTER TABLE `files`
+				ALTER TABLE `'.$prefix.'files`
 				ADD `file_storage_id` INT NOT NULL,
-				ADD INDEX (`file_storage_id`),
-				ADD FOREIGN KEY (`file_storage_id`) REFERENCES `filebin_test`.`file_storage`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;
+				ADD INDEX (`file_storage_id`);
 			');
 
 			$this->db->query('
-				INSERT INTO file_storage (storage-id, filesize, mimetype)
-				SELECT hash, filesize, mimetype FROM files;
+				INSERT INTO `'.$prefix.'file_storage` (id, filesize, mimetype, hash, date)
+				SELECT NULL, filesize, mimetype, hash, date FROM `'.$prefix.'files`;
 			');
 
 			$this->db->query('
-				UPDATE files f
-				JOIN file_storage fs ON fs.data_id = f.hash
+				UPDATE `'.$prefix.'files` f
+				JOIN `'.$prefix.'file_storage` fs ON fs.hash = f.hash
 				SET f.file_storage_id = fs.id
 			');
 
-			$this->dbforge->drop_column("files", array("hash", "mimetype", "filesize"));
+			$chunk = 500;
+			$total = $this->db->count_all("file_storage");
+
+			for ($limit = 0; $limit < $total; $limit += $chunk) {
+				$query = $this->db->select('hash, id')
+					->from('file_storage')
+					->limit($chunk, $limit)
+					->get()->result_array();
+
+				foreach ($query as $key => $item) {
+					$old = $this->mfile->file($item["hash"]);
+					$data_id = $item["hash"].'-'.$item["id"];
+					$new = $this->mfile->file($data_id);
+					rename($old, $new);
+				}
+			}
+
+			$this->db->query('
+				ALTER TABLE `'.$prefix.'files`
+				ADD FOREIGN KEY (`file_storage_id`) REFERENCES `'.$prefix.'file_storage`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;
+			');
+
+			$this->dbforge->drop_column("files", "hash");
+			$this->dbforge->drop_column("files", "mimetype");
+			$this->dbforge->drop_column("files", "filesize");
 		}
 	}
 
diff --git a/application/models/mfile.php b/application/models/mfile.php
index 9af0ed497..32ec224f0 100644
--- a/application/models/mfile.php
+++ b/application/models/mfile.php
@@ -62,55 +62,48 @@ class Mfile extends CI_Model {
 		}
 	}
 
-	public function stale_hash($hash)
-	{
-		return $this->unused_file($hash);
-	}
-
 	function get_filedata($id)
 	{
-		return cache_function("filedata-$id", 300, function() use ($id) {
+		return cache_function("filedatav2-$id", 300, function() use ($id) {
 			$query = $this->db
-				->select('id, hash, filename, mimetype, date, user, filesize')
+				->select('files.id, hash, file_storage.id storage_id, filename, mimetype, files.date, user, filesize')
 				->from('files')
-				->where('id', $id)
+				->join('file_storage', 'file_storage.id = files.file_storage_id')
+				->where('files.id', $id)
 				->limit(1)
 				->get();
 
 			if ($query->num_rows() > 0) {
-				return $query->row_array();
+				$data = $query->row_array();
+				$data["data_id"] = $data["hash"]."-".$data["storage_id"];
+				return $data;
 			} else {
 				return false;
 			}
 		});
 	}
 
-	// return the folder in which the file with $hash is stored
-	function folder($hash) {
-		return $this->config->item('upload_path').'/'.substr($hash, 0, 3);
+	// return the folder in which the file with $data_id is stored
+	function folder($data_id) {
+		return $this->config->item('upload_path').'/'.substr($data_id, 0, 3);
 	}
 
-	// Returns the full path to the file with $hash
-	function file($hash) {
-		return $this->folder($hash).'/'.$hash;
+	// Returns the full path to the file with $data_id
+	function file($data_id) {
+		return $this->folder($data_id).'/'.$data_id;
 	}
 
-	// Add a hash to the DB
-	function add_file($hash, $id, $filename)
+	// Add a file to the DB
+	function add_file($id, $filename, $storage_id)
 	{
 		$userid = $this->muser->get_userid();
 
-		$mimetype = mimetype($this->file($hash));
-
-		$filesize = filesize($this->file($hash));
 		$this->db->insert("files", array(
 			"id" => $id,
-			"hash" => $hash,
 			"filename" => $filename,
 			"date" => time(),
 			"user" => $userid,
-			"mimetype" => $mimetype,
-			"filesize" => $filesize,
+			"file_storage_id" => $storage_id,
 		));
 	}
 
@@ -160,7 +153,7 @@ class Mfile extends CI_Model {
 	public function get_timeout($id)
 	{
 		$filedata = $this->get_filedata($id);
-		$file = $this->file($filedata["hash"]);
+		$file = $this->file($filedata["data_id"]);
 
 		if ($this->config->item("upload_max_age") == 0) {
 			return -1;
@@ -184,11 +177,14 @@ class Mfile extends CI_Model {
 		}
 	}
 
-	private function unused_file($hash)
+	private function unused_file($data_id)
 	{
-		$query = $this->db->select('id')
+		list ($hash, $storage_id) = explode("-", $data_id);
+		$query = $this->db->select('files.id')
 			->from('files')
+			->join('file_storage', 'file_storage.id = files.file_storage_id')
 			->where('hash', $hash)
+			->where('file_storage.id', $storage_id)
 			->limit(1)
 			->get();
 
@@ -228,31 +224,28 @@ class Mfile extends CI_Model {
 		}
 
 		if ($filedata !== false) {
-			assert(isset($filedata["hash"]));
-			if ($this->unused_file($filedata['hash'])) {
-				if (file_exists($this->file($filedata['hash']))) {
-					unlink($this->file($filedata['hash']));
-				}
-				$dir = $this->folder($filedata['hash']);
-				if (file_exists($dir)) {
-					if (count(scandir($dir)) == 2) {
-						rmdir($dir);
-					}
-				}
+			assert(isset($filedata["data_id"]));
+			if ($this->unused_file($filedata['data_id'])) {
+				$this->delete_data_id($filedata['data_id']);
 			}
 		}
 		return true;
 	}
 
-	public function delete_hash($hash)
+	public function delete_data_id($data_id)
 	{
-		$ids = $this->db->select('id')
-			->from('files')
-			->where('hash', $hash)
-			->get()->result_array();
-
-		foreach ($ids as $entry) {
-			$this->delete_id($entry["id"]);
+		list ($hash, $storage_id) = explode("-", $data_id);
+		
+		$this->db->where('id', $storage_id)
+			->delete('file_storage');
+		if (file_exists($this->file($data_id))) {
+			unlink($this->file($data_id));
+		}
+		$dir = $this->folder($data_id);
+		if (file_exists($dir)) {
+			if (count(scandir($dir)) == 2) {
+				rmdir($dir);
+			}
 		}
 	}
 
diff --git a/application/service/files.php b/application/service/files.php
index 33354d456..ef1de2911 100644
--- a/application/service/files.php
+++ b/application/service/files.php
@@ -17,9 +17,10 @@ class files {
 		$items = array();
 
 		// TODO: thumbnail urls where available
-		$fields = array("id", "filename", "mimetype", "date", "hash", "filesize");
+		$fields = array("files.id", "filename", "mimetype", "files.date", "hash", "filesize");
 		$query = $CI->db->select(implode(',', $fields))
 			->from('files')
+			->join('file_storage', 'file_storage.id = files.file_storage_id')
 			->where('user', $user)
 			->get()->result_array();
 		foreach ($query as $key => $item) {
@@ -30,7 +31,8 @@ class files {
 			SELECT coalesce(sum(filesize), 0) sum
 			FROM (
 				SELECT DISTINCT hash, filesize
-				FROM `".$CI->db->dbprefix."files`
+				FROM `".$CI->db->dbprefix."file_storage`
+				JOIN `".$CI->db->dbprefix."files` ON `".$CI->db->dbprefix."file_storage`.id = `".$CI->db->dbprefix."files`.file_storage_id
 				WHERE user = ?
 			) sub
 			", array($user))->row_array();
@@ -79,48 +81,65 @@ class files {
 
 	static public function add_file_data($id, $content, $filename)
 	{
-		self::add_file_callback($id, $filename, array(
-			"hash" => function() use ($content) {
-				return md5($content);
-			},
-			"data_writer" => function($dest) use ($content) {
-				file_put_contents($dest, $content);
-			}
-		));
+		$f = new \libraries\Tempfile();
+		$file = $f->get_file();
+		file_put_contents($file, $content);
+		self::add_file_callback($id, $file, $filename);
 	}
 
 	static public function add_uploaded_file($id, $file, $filename)
 	{
-		self::add_file_callback($id, $filename, array(
-			"hash" => function() use ($file) {
-				return md5_file($file);
-			},
-			"data_writer" => function($dest) use ($file) {
-				move_uploaded_file($file, $dest);
-			}
-		));
+		self::add_file_callback($id, $file, $filename);
 	}
 
-	// TODO: an interface would be nice here, but php doesn't support anonymous
-	// objects so let's use an array for now
-	static private function add_file_callback($id, $filename, $callbacks)
+	static private function add_file_callback($id, $new_file, $filename)
 	{
-		assert(isset($callbacks["hash"]));
-		assert(isset($callbacks["data_writer"]));
-
 		$CI =& get_instance();
-		$hash = $callbacks["hash"]();
+		$hash = md5_file($new_file);
+		$storage_id = null;
+
+		$query = $CI->db->select('id, hash')
+			->from('file_storage')
+			->where('hash', $hash)
+			->get()->result_array();
+
+		foreach($query as $row) {
+			$data_id = implode("-", array($row['hash'], $row['id']));
+			$old_file = $CI->mfile->file($data_id);
+
+			// TODO: set $new_file
+			if (files_are_equal($old_file, $new_file)) {
+				$storage_id = $row["id"];
+				break;
+			}
+		}
+
+		if ($storage_id === null) {
+			$filesize = filesize($new_file);
+			$mimetype = mimetype($new_file);
+
+			$CI->db->insert("file_storage", array(
+				"filesize" => $filesize,
+				"mimetype" => $mimetype,
+				"hash" => $hash,
+				"date" => time(),
+			));
+			$storage_id = $CI->db->insert_id();
+		}
+		$data_id = $hash."-".$storage_id;
 
-		$dir = $CI->mfile->folder($hash);
+		// TODO: all this doesn't have to run if the file exists. updating the mtime would be enough
+		//       that would also be better for COW filesystems
+		$dir = $CI->mfile->folder($data_id);
 		file_exists($dir) || mkdir ($dir);
-		$new_path = $CI->mfile->file($hash);
+		$new_path = $CI->mfile->file($data_id);
 
 		$dest = new \service\storage($new_path);
 		$tmpfile = $dest->begin();
-		$callbacks["data_writer"]($tmpfile);
+		rename($new_file, $tmpfile);
 		$dest->commit();
 
-		$CI->mfile->add_file($hash, $id, $filename);
+		$CI->mfile->add_file($id, $filename, $storage_id);
 	}
 
 	static public function verify_uploaded_files($files)
@@ -293,7 +312,7 @@ class files {
 
 	static public function valid_id(array $filedata, array $config, $model, $current_date)
 	{
-		assert(isset($filedata["hash"]));
+		assert(isset($filedata["data_id"]));
 		assert(isset($filedata["id"]));
 		assert(isset($filedata["user"]));
 		assert(isset($filedata["date"]));
@@ -301,10 +320,10 @@ class files {
 		assert(isset($config["sess_expiration"]));
 		assert(isset($config["small_upload_size"]));
 
-		$file = $model->file($filedata['hash']);
+		$file = $model->file($filedata['data_id']);
 
 		if (!$model->file_exists($file)) {
-			$model->delete_hash($filedata["hash"]);
+			$model->delete_data_id($filedata["data_id"]);
 			return false;
 		}
 
@@ -331,7 +350,7 @@ class files {
 			// of the last upload
 			$mtime = $model->filemtime($file);
 			if ($mtime < $remove_before) {
-				$model->delete_hash($filedata["hash"]);
+				$model->delete_data_id($filedata["data_id"]);
 			} else {
 				$model->delete_id($filedata["id"]);
 			}
diff --git a/application/tests/test_service_files_valid_id.php b/application/tests/test_service_files_valid_id.php
index 436624336..c090dcbd9 100644
--- a/application/tests/test_service_files_valid_id.php
+++ b/application/tests/test_service_files_valid_id.php
@@ -28,13 +28,14 @@ class test_service_files_valid_id extends Test {
 	{
 		$this->model = \Mockery::mock("Mfile");
 		$this->model->shouldReceive("delete_id")->never()->byDefault();
-		$this->model->shouldReceive("delete_hash")->never()->byDefault();
-		$this->model->shouldReceive("file")->with("file-hash-1")->andReturn("/invalid/path/file-1")->byDefault();
+		$this->model->shouldReceive("delete_data_id")->never()->byDefault();
+		$this->model->shouldReceive("file")->with("file-hash-1-1")->andReturn("/invalid/path/file-1")->byDefault();
 		$this->model->shouldReceive("filemtime")->with("/invalid/path/file-1")->andReturn(500)->byDefault();
 		$this->model->shouldReceive("filesize")->with("/invalid/path/file-1")->andReturn(50*1024)->byDefault();
 		$this->model->shouldReceive("file_exists")->with("/invalid/path/file-1")->andReturn(true)->byDefault();
 
 		$this->filedata = array(
+			"data_id" => "file-hash-1-1",
 			"hash" => "file-hash-1",
 			"id" => "file-id-1",
 			"user" => 2,
@@ -69,7 +70,7 @@ class test_service_files_valid_id extends Test {
 
 	public function test_valid_id_removeOldFile()
 	{
-		$this->model->shouldReceive("delete_hash")->with("file-hash-1")->once();
+		$this->model->shouldReceive("delete_data_id")->with("file-hash-1-1")->once();
 
 		$ret = \service\files::valid_id($this->filedata, $this->config, $this->model, 550);
 		$this->t->is($ret, false, "file is old and should be removed");
@@ -104,7 +105,7 @@ class test_service_files_valid_id extends Test {
 	public function test_valid_id_removeMissingFile()
 	{
 		$this->model->shouldReceive("file_exists")->with("/invalid/path/file-1")->once()->andReturn(false);
-		$this->model->shouldReceive("delete_hash")->with("file-hash-1")->once();
+		$this->model->shouldReceive("delete_data_id")->with("file-hash-1-1")->once();
 
 		$ret = \service\files::valid_id($this->filedata, $this->config, $this->model, 505);
 		$this->t->is($ret, false, "missing file should be removed");