From b6a9e16bcd799abb3fecf8d406ffcde28f133f9f Mon Sep 17 00:00:00 2001 From: Ralf Becker Date: Fri, 3 Oct 2008 12:18:19 +0000 Subject: [PATCH] SQLFS (eGW's default VFS system) stores the content of the files now in a hashed directory structure based on the fs_id and not longer on the path (which can not be recovered, once the filesystem get's corrupt) --> Make backups (db AND files directory), before attempting the update !!!!!!!! --- .../inc/class.sqlfs_stream_wrapper.inc.php | 61 +++++++++++++------ phpgwapi/setup/setup.inc.php | 2 +- phpgwapi/setup/tables_update.inc.php | 38 ++++++++++++ 3 files changed, 81 insertions(+), 20 deletions(-) diff --git a/phpgwapi/inc/class.sqlfs_stream_wrapper.inc.php b/phpgwapi/inc/class.sqlfs_stream_wrapper.inc.php index 7c61359621..26449ff4c2 100644 --- a/phpgwapi/inc/class.sqlfs_stream_wrapper.inc.php +++ b/phpgwapi/inc/class.sqlfs_stream_wrapper.inc.php @@ -214,6 +214,11 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper // we buffer all write operations in a temporary file, which get's written on close $this->opened_stream = tmpfile(); } + // create the hash-dirs, if they not yet exist + elseif(!file_exists($fs_dir=dirname(self::_fs_path($this->opened_fs_id)))) + { + mkdir($fs_dir,0700,true); + } } else { @@ -252,7 +257,7 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper // do we operate directly on the filesystem if ($this->operation == self::STORE2FS) { - $this->opened_stream = fopen(self::_fs_path($path),$mode); + $this->opened_stream = fopen(self::_fs_path($this->opened_fs_id),$mode); } if ($mode[0] == 'a') // append modes: a, a+ { @@ -490,7 +495,7 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper if (($ret = $stmt->execute(array(':fs_id' => $stat['ino']))) && self::url2operation($url) == self::STORE2FS) { - unlink(self::_fs_path($path)); + unlink(self::_fs_path($stat['ino'])); } return $ret; } @@ -548,18 +553,14 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper if (self::LOG_LEVEL) error_log(__METHOD__."($url_to,$url_from) can't unlink existing $url_to!"); return false; } + unset(self::$stat_cache[$path_from]); + $stmt = self::$pdo->prepare('UPDATE '.self::TABLE.' SET fs_dir=:fs_dir,fs_name=:fs_name WHERE fs_id=:fs_id'); - if (($ret = $stmt->execute(array( + return $stmt->execute(array( ':fs_dir' => $to_dir_stat['ino'], ':fs_name' => basename($path_to), ':fs_id' => $from_stat['ino'], - ))) && $operation == self::STORE2FS) - { - rename(self::_fs_path($path_from),self::_fs_path($path_to)); - } - unset(self::$stat_cache[$path_from]); - - return $ret; + )); } /** @@ -615,7 +616,7 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper } $stmt = self::$pdo->prepare('INSERT INTO '.self::TABLE.' (fs_name,fs_dir,fs_mode,fs_uid,fs_gid,fs_size,fs_mime,fs_created,fs_modified,fs_creator'. ') VALUES (:fs_name,:fs_dir,:fs_mode,:fs_uid,:fs_gid,:fs_size,:fs_mime,:fs_created,:fs_modified,:fs_creator)'); - if (($ret = $stmt->execute(array( + return $stmt->execute(array( ':fs_name' => basename($path), ':fs_dir' => $parent['ino'], ':fs_mode' => $parent['mode'], @@ -626,11 +627,7 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper ':fs_created' => self::_pdo_timestamp(time()), ':fs_modified' => self::_pdo_timestamp(time()), ':fs_creator' => egw_vfs::$user, - ))) && self::url2operation($url) == self::STORE2FS) - { - $ok = mkdir($dir = self::_fs_path($path),0700,true); // we create all missing parent dirs, as eg. /home might not yet exist - } - return $ret; + )); } /** @@ -680,7 +677,6 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper if (($ret = $stmt->execute(array($stat['ino'])))) { self::eacl($path,null,false,$stat['ino']); // remove all (=false) evtl. existing extended acl for that dir - if (self::url2operation($url) == self::STORE2FS) rmdir(self::_fs_path($path)); } return $ret; } @@ -1378,14 +1374,32 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper return $time; } + /** + * Maximum value for a single hash element (should be 10^N): 10, 100 (default), 1000, ... + * + * DONT change this value, once you have files stored, they will no longer be found! + */ + const HASH_MAX = 100; + /** * Return the path of the stored content of a file if $this->operation == self::STORE2FS * - * @param string $path + * To limit the number of files stored in one directory, we create a hash from the fs_id: + * 1 --> /1 + * 34 --> /34 + * 123 --> /01/123 + * 4567 --> /45/4567 + * 99999 --> /09/99/99999 + * --> so one directory contains maximum 2 * HASH_MAY entries (HASH_MAX dirs + HASH_MAX files) + * @param int $id id of the file * @return string */ - static private function _fs_path($path) + static function _fs_path($id) { + if (!is_numeric($id)) + { + throw new egw_exception_wrong_parameter(__METHOD__."(id=$id) id has to be an integer!"); + } if (!isset($GLOBALS['egw_info']['server']['files_dir']) || $GLOBALS['egw_info']['server']['files_dir']) { if (is_object($GLOBALS['egw_setup']->db)) // if we run under setup, query the db for the files dir @@ -1400,6 +1414,15 @@ class sqlfs_stream_wrapper implements iface_stream_wrapper throw new egw_exception_assertion_failed("\$GLOBALS['egw_info']['server']['files_dir'] not set!"); } } + $hash = array(); + for ($n = $id; $n = (int) ($n / self::HASH_MAX); ) + { + $hash[] = sprintf('%02d',$n % self::HASH_MAX); + } + array_unshift($hash,$id); + + $path = '/sqlfs/'.implode('/',array_reverse($hash)); + //error_log(__METHOD__."($id) = '$path'"); return $GLOBALS['egw_info']['server']['files_dir'].$path; } diff --git a/phpgwapi/setup/setup.inc.php b/phpgwapi/setup/setup.inc.php index 74adcf9995..8e49a6dfeb 100755 --- a/phpgwapi/setup/setup.inc.php +++ b/phpgwapi/setup/setup.inc.php @@ -12,7 +12,7 @@ /* Basic information about this app */ $setup_info['phpgwapi']['name'] = 'phpgwapi'; $setup_info['phpgwapi']['title'] = 'eGroupWare API'; -$setup_info['phpgwapi']['version'] = '1.5.011'; +$setup_info['phpgwapi']['version'] = '1.5.012'; $setup_info['phpgwapi']['versions']['current_header'] = '1.28'; $setup_info['phpgwapi']['enable'] = 3; $setup_info['phpgwapi']['app_order'] = 1; diff --git a/phpgwapi/setup/tables_update.inc.php b/phpgwapi/setup/tables_update.inc.php index 80b1db762a..5bbf5f995f 100644 --- a/phpgwapi/setup/tables_update.inc.php +++ b/phpgwapi/setup/tables_update.inc.php @@ -432,3 +432,41 @@ function phpgwapi_upgrade1_5_010() return $GLOBALS['setup_info']['phpgwapi']['currentver'] = '1.5.011'; } +/** + * Move sqlfs files from their path based location to the new (hashed) id base one and removed the not longer used dirs + * + * @return string + */ +function phpgwapi_upgrade1_5_011() +{ + if ($GLOBALS['DEBUG']) echo "
\n";
+	egw_vfs::$is_root = true;
+	egw_vfs::load_wrapper('sqlfs');
+	egw_vfs::find('sqlfs://default/',array(
+		'url'  => true,
+		'depth' => true,
+	),create_function('$url,$stat','
+		$new_path = sqlfs_stream_wrapper::_fs_path($stat["ino"]);	// loads egw_info/server/files_dir (!)
+		if (file_exists($old_path = $GLOBALS["egw_info"]["server"]["files_dir"].($path=parse_url($url,PHP_URL_PATH))))
+		{
+			if (!is_dir($old_path))
+			{
+				if ($GLOBALS["DEBUG"]) echo "moving file $old_path --> $new_path\n";
+				if (!file_exists($parent = dirname($new_path))) mkdir($parent,0777,true);	// 777 because setup-cli might run eg. by root
+				rename($old_path,$new_path);
+			}
+			elseif($path != "/")
+			{
+				if ($GLOBALS["DEBUG"]) echo "removing directory $old_path\n";
+				rmdir($old_path);
+			}
+		}
+		else
+		{
+			echo "phpgwapi_upgrade1_5_011() $url: $old_path not found!\n";
+		}
+	'));
+	if ($GLOBALS['DEBUG']) echo "
\n"; + + return $GLOBALS['setup_info']['phpgwapi']['currentver'] = '1.5.012'; +}