diff --git a/api/setup/setup.inc.php b/api/setup/setup.inc.php index 0bf953f401..3136422d34 100644 --- a/api/setup/setup.inc.php +++ b/api/setup/setup.inc.php @@ -11,7 +11,7 @@ /* Basic information about this app */ $setup_info['api']['name'] = 'api'; $setup_info['api']['title'] = 'EGroupware API'; -$setup_info['api']['version'] = '21.1.001'; +$setup_info['api']['version'] = '21.1.003'; $setup_info['api']['versions']['current_header'] = '1.29'; // maintenance release in sync with changelog in doc/rpm-build/debian.changes $setup_info['api']['versions']['maintenance_release'] = '21.1.20211130'; @@ -140,6 +140,3 @@ $setup_info['groupdav']['author'] = $setup_info['groupdav']['maintainer'] = arra $setup_info['groupdav']['license'] = 'GPL'; $setup_info['groupdav']['hooks']['preferences'] = 'EGroupware\\Api\\CalDAV\\Hooks::menus'; $setup_info['groupdav']['hooks']['settings'] = 'EGroupware\\Api\\CalDAV\\Hooks::settings'; - - - diff --git a/api/setup/tables_current.inc.php b/api/setup/tables_current.inc.php index 648233fe5b..5e7e507294 100644 --- a/api/setup/tables_current.inc.php +++ b/api/setup/tables_current.inc.php @@ -320,7 +320,7 @@ $phpgw_baseline = array( 'fs_created' => array('type' => 'timestamp','precision' => '8','nullable' => False), 'fs_modified' => array('type' => 'timestamp','precision' => '8','nullable' => False), 'fs_mime' => array('type' => 'ascii','precision' => '96','nullable' => False), - 'fs_size' => array('type' => 'int','precision' => '8','nullable' => False), + 'fs_size' => array('type' => 'int','precision' => '8', 'default' => '0'), 'fs_creator' => array('type' => 'int','meta' => 'user','precision' => '4','nullable' => False), 'fs_modifier' => array('type' => 'int','meta' => 'user','precision' => '4'), 'fs_active' => array('type' => 'bool','nullable' => False,'default' => 't'), diff --git a/api/setup/tables_update.inc.php b/api/setup/tables_update.inc.php index 7d4893e720..5c9b0408fc 100644 --- a/api/setup/tables_update.inc.php +++ b/api/setup/tables_update.inc.php @@ -804,3 +804,36 @@ function api_upgrade21_1() return $GLOBALS['setup_info']['api']['currentver'] = '21.1.001'; } + +/** + * Allow fs_size to be NULL for quota recalculation + * + * @return string + */ +function api_upgrade21_1_001() +{ + $GLOBALS['egw_setup']->oProc->AlterColumn('egw_sqlfs','fs_size',array( + 'type' => 'int', + 'precision' => '8', + 'default' => '0' + )); + // ADOdb does not support dropping NOT NULL for PostgreSQL :( + if ($GLOBALS['egw_setup']->db->Type === 'pgsql') + { + $GLOBALS['egw_setup']->db->query('ALTER TABLE "egw_sqlfs" ALTER COLUMN "fs_size" DROP NOT NULL', __LINE__, __FILE__); + } + + return $GLOBALS['setup_info']['api']['currentver'] = '21.1.002'; +} + +/** + * Recalculate quota / directory sizes + * + * @return string + */ +function api_upgrade21_1_002() +{ + Vfs\Sqlfs\Utils::quotaRecalc(); + + return $GLOBALS['setup_info']['api']['currentver'] = '21.1.003'; +} diff --git a/api/src/Vfs/Sqlfs/StreamWrapper.php b/api/src/Vfs/Sqlfs/StreamWrapper.php index 465bcd72eb..c5a3e8f64f 100644 --- a/api/src/Vfs/Sqlfs/StreamWrapper.php +++ b/api/src/Vfs/Sqlfs/StreamWrapper.php @@ -127,6 +127,12 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface * @var int */ protected $opened_fs_id; + /** + * Initial size of opened file for adjustDirSize call + * + * @var int + */ + protected $opened_size; /** * Cache containing stat-infos from previous url_stat calls AND dir_opendir calls * @@ -326,6 +332,16 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface { $this->stream_seek(0,SEEK_END); } + // remember initial size and directory for adjustDirSize call in close + if (is_resource($this->opened_stream)) + { + $this->opened_size = empty($stat) ? $stat['size'] : 0; + if (empty($dir_stat)) + { + $dir_stat = $this->url_stat($dir,STREAM_URL_STAT_QUIET); + } + $this->opened_dir = $dir_stat['ino']; + } if (!is_resource($this->opened_stream)) error_log(__METHOD__."($url,$mode,$options) NO stream, returning false!"); return is_resource($this->opened_stream); @@ -348,10 +364,11 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface if ($this->opened_mode != 'r') { $this->stream_seek(0,SEEK_END); + $size = $this->stream_tell(); // we need to update the mime-type, size and content (if STORE2DB) $values = array( - 'fs_size' => $this->stream_tell(), + 'fs_size' => $size, // todo: analyse the file for the mime-type 'fs_mime' => Api\MimeMagic::filename2mime($this->opened_path), 'fs_id' => $this->opened_fs_id, @@ -381,6 +398,11 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface error_log(__METHOD__."() execute() failed! errorInfo()=".array2string(self::$pdo->errorInfo())); } } + // adjust directory size, if changed + if ($ret && $size != $this->opened_size && $this->opened_dir) + { + $this->adjustDirSize($this->opened_dir, $size-$this->opened_size); + } } else { @@ -584,10 +606,51 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface unset($stmt); $stmt = self::$pdo->prepare('DELETE FROM '.self::PROPS_TABLE.' WHERE fs_id=?'); $stmt->execute(array($stat['ino'])); + + if ($stat['mime'] !== self::SYMLINK_MIME_TYPE) + { + $this->adjustDirSize($parent_stat['ino'], -$stat['size']); + } } return $ret; } + /** + * Adjust directory sizes + * + * Adjustment is always relative, so concurrency does not matter. + * Adjustment is made to all parent directories too! + * + * @param int $fs_id fs_id of directory to adjust + * @param int $fs_size size adjustment + * @param bool $fs_id_is_dir=true false: $fs_id is the file causing the change (only adjust its directories) + */ + protected function adjustDirSize(int $fs_id, int $fs_size, bool $fs_id_is_dir=true) + { + if (!$fs_size) return; // nothing to do + + static $stmt=null,$parent_stmt; + if (!isset($stmt)) + { + $stmt = self::$pdo->prepare('UPDATE '.self::TABLE.' SET fs_size=fs_size+:fs_size WHERE fs_id=:fs_id'); + $parent_stmt = self::$pdo->prepare('SELECT fs_dir FROM '.self::TABLE.' WHERE fs_id=:fs_id'); + } + + $max_depth = 100; + do + { + if ($fs_id_is_dir || $max_depth < 100) + { + $stmt->execute([ + 'fs_id' => $fs_id, + 'fs_size' => $fs_size, + ]); + } + $parent_stmt->execute(['fs_id' => $fs_id]); + } + while (($fs_id = $parent_stmt->fetchColumn()) && --$max_depth > 0); + } + /** * This method is called in response to rename() calls on URL paths associated with the wrapper. * @@ -667,6 +730,12 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface )); unset(self::$stat_cache[$path_to]); } + + if ($ok && $to_dir_stat['ino'] !== $from_dir_stat['ino'] && $new_mime !== self::SYMLINK_MIME_TYPE) + { + $this->adjustDirSize($from_dir_stat['ino'], -$from_stat['size']); + $this->adjustDirSize($to_dir_stat['ino'], $from_stat['size']); + } return $ok; } @@ -1390,7 +1459,7 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface 'fs_modified' => self::_pdo_timestamp(time()), 'fs_creator' => Vfs::$user, 'fs_mime' => self::SYMLINK_MIME_TYPE, - 'fs_size' => bytes($target), + 'fs_size' => 0, 'fs_link' => $target, )); } diff --git a/api/src/Vfs/Sqlfs/Utils.php b/api/src/Vfs/Sqlfs/Utils.php index ac1d02f72e..b1edf4fd32 100644 --- a/api/src/Vfs/Sqlfs/Utils.php +++ b/api/src/Vfs/Sqlfs/Utils.php @@ -103,7 +103,7 @@ class Utils extends StreamWrapper } /** - * Check and optionaly fix corruption in sqlfs + * Check and optionally fix corruption in sqlfs * * @param boolean $check_only =true * @return array with messages / found problems @@ -126,12 +126,17 @@ class Utils extends StreamWrapper } foreach (Api\Hooks::process(array( - 'location' => 'fsck', - 'check_only' => $check_only) + 'location' => 'fsck', + 'check_only' => $check_only) ) as $app_msgs) { if ($app_msgs) $msgs = array_merge($msgs, $app_msgs); } + + // also run quota recalc as fsck might have (re-)moved files + list($dirs, $iterations, $time) = Vfs\Sqlfs\Utils::quotaRecalc(); + $msgs[] = lang("Recalculated %1 directories in %2 iterations and %3 seconds", $dirs, $iterations, number_format($time, 1)); + return $msgs; } @@ -501,6 +506,52 @@ class Utils extends StreamWrapper } return $msgs; } + + /** + * Recalculate directory sizes + * + * @return int[] directories recalculated, iterations necessary, time + */ + static function quotaRecalc() + { + if (!is_object(self::$pdo)) + { + self::_pdo(); + } + $start = microtime(true); + $table = self::TABLE; + self::$pdo->query("UPDATE $table SET fs_size=NULL WHERE fs_mime='".self::DIR_MIME_TYPE."' AND fs_active"); + self::$pdo->query("UPDATE $table SET fs_size=0 WHERE fs_mime='".self::SYMLINK_MIME_TYPE."'"); + + $stmt = self::$pdo->prepare("SELECT $table.fs_id, SUM(child.fs_size) as total +FROM $table +LEFT JOIN $table child ON $table.fs_id=child.fs_dir AND child.fs_active +WHERE $table.fs_active AND $table.fs_mime='httpd/unix-directory' AND $table.fs_size IS NULL +GROUP BY $table.fs_id +HAVING COUNT(child.fs_id)=0 OR COUNT(CASE WHEN child.fs_size IS NULL THEN 1 ELSE NULL END)=0 +ORDER BY $table.fs_id DESC +LIMIT 500"); + $stmt->setFetchMode(PDO::FETCH_ASSOC); + $update = self::$pdo->prepare("UPDATE $table SET fs_size=:fs_size WHERE fs_id=:fs_id"); + $iterations = $dirs = 0; + do + { + $rows = 0; + $stmt->execute(); + foreach ($stmt as $row) + { + $update->execute([ + 'fs_size' => $row['total'] ?? 0, + 'fs_id' => $row['fs_id'], + ]); + ++$rows; + } + $dirs += $rows; + } + while ($rows > 0 && ++$iterations < 100); + + return [$dirs, $iterations, microtime(true)-$start]; + } } // fsck testcode, if this file is called via it's URL (you need to uncomment it!) diff --git a/filemanager/inc/class.filemanager_admin.inc.php b/filemanager/inc/class.filemanager_admin.inc.php index 8d4269fa5f..47d4f1d468 100644 --- a/filemanager/inc/class.filemanager_admin.inc.php +++ b/filemanager/inc/class.filemanager_admin.inc.php @@ -29,6 +29,7 @@ class filemanager_admin extends filemanager_ui public $public_functions = array( 'index' => true, 'fsck' => true, + 'quotaRecalc' => true, ); /** @@ -365,4 +366,14 @@ class filemanager_admin extends filemanager_ui $GLOBALS['egw']->framework->render($content, lang('Admin').' - '.lang('Check virtual filesystem'), true); } + + /** + * Recalculate directory sizes + */ + function quotaRecalc() + { + list($dirs, $iterations, $time) = Vfs\Sqlfs\Utils::quotaRecalc(); + + echo lang("Recalculated %1 directories in %2 iterations and %3 seconds", $dirs, $iterations, number_format($time, 1))."\n"; + } } \ No newline at end of file diff --git a/filemanager/inc/class.filemanager_hooks.inc.php b/filemanager/inc/class.filemanager_hooks.inc.php index 2ea5bd1b03..8b14afe77a 100644 --- a/filemanager/inc/class.filemanager_hooks.inc.php +++ b/filemanager/inc/class.filemanager_hooks.inc.php @@ -97,6 +97,7 @@ class filemanager_hooks //'Site Configuration' => Egw::link('/index.php','menuaction=admin.admin_config.index&appname='.self::$appname.'&ajax=true'), 'Custom fields' => Egw::link('/index.php','menuaction=admin.admin_customfields.index&appname='.self::$appname.'&ajax=true'), 'Check virtual filesystem' => Egw::link('/index.php','menuaction=filemanager.filemanager_admin.fsck'), + 'Recalculate quota' => Egw::link('/index.php','menuaction=filemanager.filemanager_admin.quotaRecalc'), 'VFS mounts and versioning' => Egw::link('/index.php', 'menuaction=filemanager.filemanager_admin.index&ajax=true'), ); if ($location == 'admin')