* Filemanager/API: directory size is now the sum of the size of its contained files (allowing to eg. implement quota in future)

api/setup/setup.inc.php

@@ -11,7 +11,7 @@
 /* Basic information about this app */
 $setup_info['api']['name']      = 'api';
 $setup_info['api']['title']     = 'EGroupware API';
-$setup_info['api']['version']   = '21.1.001';
+$setup_info['api']['version']   = '21.1.003';
 $setup_info['api']['versions']['current_header'] = '1.29';
 // maintenance release in sync with changelog in doc/rpm-build/debian.changes
 $setup_info['api']['versions']['maintenance_release'] = '21.1.20211130';
@@ -140,6 +140,3 @@ $setup_info['groupdav']['author'] = $setup_info['groupdav']['maintainer'] = arra
 $setup_info['groupdav']['license'] = 'GPL';
 $setup_info['groupdav']['hooks']['preferences']	= 'EGroupware\\Api\\CalDAV\\Hooks::menus';
 $setup_info['groupdav']['hooks']['settings']	= 'EGroupware\\Api\\CalDAV\\Hooks::settings';
-
-
-

api/setup/tables_current.inc.php

@@ -320,7 +320,7 @@ $phpgw_baseline = array(
 			'fs_created' => array('type' => 'timestamp','precision' => '8','nullable' => False),
 			'fs_modified' => array('type' => 'timestamp','precision' => '8','nullable' => False),
 			'fs_mime' => array('type' => 'ascii','precision' => '96','nullable' => False),
-			'fs_size' => array('type' => 'int','precision' => '8','nullable' => False),
+			'fs_size' => array('type' => 'int','precision' => '8', 'default' => '0'),
 			'fs_creator' => array('type' => 'int','meta' => 'user','precision' => '4','nullable' => False),
 			'fs_modifier' => array('type' => 'int','meta' => 'user','precision' => '4'),
 			'fs_active' => array('type' => 'bool','nullable' => False,'default' => 't'),

api/setup/tables_update.inc.php

@@ -804,3 +804,36 @@ function api_upgrade21_1()
 
 	return $GLOBALS['setup_info']['api']['currentver'] = '21.1.001';
 }
+
+/**
+ * Allow fs_size to be NULL for quota recalculation
+ *
+ * @return string
+ */
+function api_upgrade21_1_001()
+{
+	$GLOBALS['egw_setup']->oProc->AlterColumn('egw_sqlfs','fs_size',array(
+		'type' => 'int',
+		'precision' => '8',
+		'default' => '0'
+	));
+	// ADOdb does not support dropping NOT NULL for PostgreSQL :(
+	if ($GLOBALS['egw_setup']->db->Type === 'pgsql')
+	{
+		$GLOBALS['egw_setup']->db->query('ALTER TABLE "egw_sqlfs" ALTER COLUMN "fs_size" DROP NOT NULL', __LINE__, __FILE__);
+	}
+
+	return $GLOBALS['setup_info']['api']['currentver'] = '21.1.002';
+}
+
+/**
+ * Recalculate quota / directory sizes
+ *
+ * @return string
+ */
+function api_upgrade21_1_002()
+{
+	Vfs\Sqlfs\Utils::quotaRecalc();
+
+	return $GLOBALS['setup_info']['api']['currentver'] = '21.1.003';
+}

api/src/Vfs/Sqlfs/StreamWrapper.php

@@ -127,6 +127,12 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 	 * @var int
 	 */
 	protected $opened_fs_id;
+	/**
+	 * Initial size of opened file for adjustDirSize call
+	 *
+	 * @var int
+	 */
+	protected $opened_size;
 	/**
 	 * Cache containing stat-infos from previous url_stat calls AND dir_opendir calls
 	 *
@@ -326,6 +332,16 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 		{
 			$this->stream_seek(0,SEEK_END);
 		}
+		// remember initial size and directory for adjustDirSize call in close
+		if (is_resource($this->opened_stream))
+		{
+			$this->opened_size = empty($stat) ? $stat['size'] : 0;
+			if (empty($dir_stat))
+			{
+				$dir_stat = $this->url_stat($dir,STREAM_URL_STAT_QUIET);
+			}
+			$this->opened_dir = $dir_stat['ino'];
+		}
 		if (!is_resource($this->opened_stream)) error_log(__METHOD__."($url,$mode,$options) NO stream, returning false!");
 
 		return is_resource($this->opened_stream);
@@ -348,10 +364,11 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 		if ($this->opened_mode != 'r')
 		{
 			$this->stream_seek(0,SEEK_END);
+			$size = $this->stream_tell();
 
 			// we need to update the mime-type, size and content (if STORE2DB)
 			$values = array(
-				'fs_size' => $this->stream_tell(),
+				'fs_size' => $size,
 				// todo: analyse the file for the mime-type
 				'fs_mime' => Api\MimeMagic::filename2mime($this->opened_path),
 				'fs_id'   => $this->opened_fs_id,
@@ -381,6 +398,11 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 					error_log(__METHOD__."() execute() failed! errorInfo()=".array2string(self::$pdo->errorInfo()));
 				}
 			}
+			// adjust directory size, if changed
+			if ($ret && $size != $this->opened_size && $this->opened_dir)
+			{
+				$this->adjustDirSize($this->opened_dir, $size-$this->opened_size);
+			}
 		}
 		else
 		{
@@ -584,10 +606,51 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 			unset($stmt);
 			$stmt = self::$pdo->prepare('DELETE FROM '.self::PROPS_TABLE.' WHERE fs_id=?');
 			$stmt->execute(array($stat['ino']));
+
+			if ($stat['mime'] !== self::SYMLINK_MIME_TYPE)
+			{
+				$this->adjustDirSize($parent_stat['ino'], -$stat['size']);
+			}
 		}
 		return $ret;
 	}
 
+	/**
+	 * Adjust directory sizes
+	 *
+	 * Adjustment is always relative, so concurrency does not matter.
+	 * Adjustment is made to all parent directories too!
+	 *
+	 * @param int $fs_id fs_id of directory to adjust
+	 * @param int $fs_size size adjustment
+	 * @param bool $fs_id_is_dir=true false: $fs_id is the file causing the change (only adjust its directories)
+	 */
+	protected function adjustDirSize(int $fs_id, int $fs_size, bool $fs_id_is_dir=true)
+	{
+		if (!$fs_size) return;  // nothing to do
+
+		static $stmt=null,$parent_stmt;
+		if (!isset($stmt))
+		{
+			$stmt = self::$pdo->prepare('UPDATE '.self::TABLE.' SET fs_size=fs_size+:fs_size WHERE fs_id=:fs_id');
+			$parent_stmt = self::$pdo->prepare('SELECT fs_dir FROM '.self::TABLE.' WHERE fs_id=:fs_id');
+		}
+
+		$max_depth = 100;
+		do
+		{
+			if ($fs_id_is_dir || $max_depth < 100)
+			{
+				$stmt->execute([
+					'fs_id' => $fs_id,
+					'fs_size' => $fs_size,
+				]);
+			}
+			$parent_stmt->execute(['fs_id' => $fs_id]);
+		}
+		while (($fs_id = $parent_stmt->fetchColumn()) && --$max_depth > 0);
+	}
+
 	/**
 	 * This method is called in response to rename() calls on URL paths associated with the wrapper.
 	 *
@@ -667,6 +730,12 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 			));
 			unset(self::$stat_cache[$path_to]);
 		}
+
+		if ($ok && $to_dir_stat['ino'] !== $from_dir_stat['ino'] && $new_mime !== self::SYMLINK_MIME_TYPE)
+		{
+			$this->adjustDirSize($from_dir_stat['ino'], -$from_stat['size']);
+			$this->adjustDirSize($to_dir_stat['ino'], $from_stat['size']);
+		}
 		return $ok;
 	}
 
@@ -1390,7 +1459,7 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
 			'fs_modified' => self::_pdo_timestamp(time()),
 			'fs_creator'  => Vfs::$user,
 			'fs_mime'     => self::SYMLINK_MIME_TYPE,
-			'fs_size'     => bytes($target),
+			'fs_size'     => 0,
 			'fs_link'     => $target,
 		));
 	}

api/src/Vfs/Sqlfs/Utils.php

@@ -103,7 +103,7 @@ class Utils extends StreamWrapper
 	}
 
 	/**
-	 * Check and optionaly fix corruption in sqlfs
+	 * Check and optionally fix corruption in sqlfs
 	 *
 	 * @param boolean $check_only =true
 	 * @return array with messages / found problems
@@ -126,12 +126,17 @@ class Utils extends StreamWrapper
 		}
 
 		foreach (Api\Hooks::process(array(
-			'location' => 'fsck',
-			'check_only' => $check_only)
+				'location' => 'fsck',
+				'check_only' => $check_only)
 		) as $app_msgs)
 		{
 			if ($app_msgs) $msgs = array_merge($msgs, $app_msgs);
 		}
+
+		// also run quota recalc as fsck might have (re-)moved files
+		list($dirs, $iterations, $time) = Vfs\Sqlfs\Utils::quotaRecalc();
+		$msgs[] = lang("Recalculated %1 directories in %2 iterations and %3 seconds", $dirs, $iterations, number_format($time, 1));
+
 		return $msgs;
 	}
 
@@ -501,6 +506,52 @@ class Utils extends StreamWrapper
 		}
 		return $msgs;
 	}
+
+	/**
+	 * Recalculate directory sizes
+	 *
+	 * @return int[] directories recalculated, iterations necessary, time
+	 */
+	static function quotaRecalc()
+	{
+		if (!is_object(self::$pdo))
+		{
+			self::_pdo();
+		}
+		$start = microtime(true);
+		$table = self::TABLE;
+		self::$pdo->query("UPDATE $table SET fs_size=NULL WHERE fs_mime='".self::DIR_MIME_TYPE."' AND fs_active");
+		self::$pdo->query("UPDATE $table SET fs_size=0 WHERE fs_mime='".self::SYMLINK_MIME_TYPE."'");
+
+		$stmt = self::$pdo->prepare("SELECT $table.fs_id, SUM(child.fs_size) as total
+FROM $table
+LEFT JOIN $table child ON $table.fs_id=child.fs_dir AND child.fs_active
+WHERE $table.fs_active AND $table.fs_mime='httpd/unix-directory' AND $table.fs_size IS NULL
+GROUP BY $table.fs_id
+HAVING COUNT(child.fs_id)=0 OR COUNT(CASE WHEN child.fs_size IS NULL THEN 1 ELSE NULL END)=0
+ORDER BY $table.fs_id DESC
+LIMIT 500");
+		$stmt->setFetchMode(PDO::FETCH_ASSOC);
+		$update = self::$pdo->prepare("UPDATE $table SET fs_size=:fs_size WHERE fs_id=:fs_id");
+		$iterations = $dirs = 0;
+		do
+		{
+			$rows = 0;
+			$stmt->execute();
+			foreach ($stmt as $row)
+			{
+				$update->execute([
+					'fs_size' => $row['total'] ?? 0,
+					'fs_id' => $row['fs_id'],
+				]);
+				++$rows;
+			}
+			$dirs += $rows;
+		}
+		while ($rows > 0 && ++$iterations < 100);
+
+		return [$dirs, $iterations, microtime(true)-$start];
+	}
 }
 
 // fsck testcode, if this file is called via it's URL (you need to uncomment it!)

filemanager/inc/class.filemanager_admin.inc.php

@@ -29,6 +29,7 @@ class filemanager_admin extends filemanager_ui
 	public $public_functions = array(
 		'index' => true,
 		'fsck' => true,
+		'quotaRecalc' => true,
 	);
 
 	/**
@@ -365,4 +366,14 @@ class filemanager_admin extends filemanager_ui
 
 		$GLOBALS['egw']->framework->render($content, lang('Admin').' - '.lang('Check virtual filesystem'), true);
 	}
+
+	/**
+	 * Recalculate directory sizes
+	 */
+	function quotaRecalc()
+	{
+		list($dirs, $iterations, $time) = Vfs\Sqlfs\Utils::quotaRecalc();
+
+		echo lang("Recalculated %1 directories in %2 iterations and %3 seconds", $dirs, $iterations, number_format($time, 1))."\n";
+	}
 }

filemanager/inc/class.filemanager_hooks.inc.php

@@ -97,6 +97,7 @@ class filemanager_hooks
 			//'Site Configuration' => Egw::link('/index.php','menuaction=admin.admin_config.index&appname='.self::$appname.'&ajax=true'),
 			'Custom fields' => Egw::link('/index.php','menuaction=admin.admin_customfields.index&appname='.self::$appname.'&ajax=true'),
 			'Check virtual filesystem' => Egw::link('/index.php','menuaction=filemanager.filemanager_admin.fsck'),
+			'Recalculate quota' => Egw::link('/index.php','menuaction=filemanager.filemanager_admin.quotaRecalc'),
 			'VFS mounts and versioning' => Egw::link('/index.php', 'menuaction=filemanager.filemanager_admin.index&ajax=true'),
 		);
 		if ($location == 'admin')