extern/egroupware_official
1
0
Fork 0
mirror of https://github.com/EGroupware/egroupware.git synced 2024-11-29 03:13:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

* Filemanager/API: directory size is now the sum of the size of its contained files (allowing to eg. implement quota in future)

Browse Source
This commit is contained in:
Ralf Becker 2021-12-30 14:37:58 +02:00
parent 578accd72f
commit e2c09aa1b0
7 changed files with 172 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/setup/setup.inc.php
View File

@ -11,7 +11,7 @@
/* Basic information about this app */
$setup_info['api']['name'] = 'api';
$setup_info['api']['title'] = 'EGroupware API';
$setup_info['api']['version'] = '21.1.001';
$setup_info['api']['version'] = '21.1.003';
$setup_info['api']['versions']['current_header'] = '1.29';
// maintenance release in sync with changelog in doc/rpm-build/debian.changes
$setup_info['api']['versions']['maintenance_release'] = '21.1.20211130';
@ -140,6 +140,3 @@ $setup_info['groupdav']['author'] = $setup_info['groupdav']['maintainer'] = arra
$setup_info['groupdav']['license'] = 'GPL';
$setup_info['groupdav']['hooks']['preferences'] = 'EGroupware\\Api\\CalDAV\\Hooks::menus';
$setup_info['groupdav']['hooks']['settings'] = 'EGroupware\\Api\\CalDAV\\Hooks::settings';

2
api/setup/tables_current.inc.php
View File

@ -320,7 +320,7 @@ $phpgw_baseline = array(
'fs_created' => array('type' => 'timestamp','precision' => '8','nullable' => False),
'fs_modified' => array('type' => 'timestamp','precision' => '8','nullable' => False),
'fs_mime' => array('type' => 'ascii','precision' => '96','nullable' => False),
'fs_size' => array('type' => 'int','precision' => '8','nullable' => False),
'fs_size' => array('type' => 'int','precision' => '8', 'default' => '0'),
'fs_creator' => array('type' => 'int','meta' => 'user','precision' => '4','nullable' => False),
'fs_modifier' => array('type' => 'int','meta' => 'user','precision' => '4'),
'fs_active' => array('type' => 'bool','nullable' => False,'default' => 't'),

33
api/setup/tables_update.inc.php
View File

@ -804,3 +804,36 @@ function api_upgrade21_1()
return $GLOBALS['setup_info']['api']['currentver'] = '21.1.001';
}
/**
* Allow fs_size to be NULL for quota recalculation
*
* @return string
*/
function api_upgrade21_1_001()
{
$GLOBALS['egw_setup']->oProc->AlterColumn('egw_sqlfs','fs_size',array(
'type' => 'int',
'precision' => '8',
'default' => '0'
));
// ADOdb does not support dropping NOT NULL for PostgreSQL :(
if ($GLOBALS['egw_setup']->db->Type === 'pgsql')
{
$GLOBALS['egw_setup']->db->query('ALTER TABLE "egw_sqlfs" ALTER COLUMN "fs_size" DROP NOT NULL', __LINE__, __FILE__);
}
return $GLOBALS['setup_info']['api']['currentver'] = '21.1.002';
}
/**
* Recalculate quota / directory sizes
*
* @return string
*/
function api_upgrade21_1_002()
{
Vfs\Sqlfs\Utils::quotaRecalc();
return $GLOBALS['setup_info']['api']['currentver'] = '21.1.003';
}

73
api/src/Vfs/Sqlfs/StreamWrapper.php
View File

@ -127,6 +127,12 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
* @var int
*/
protected $opened_fs_id;
/**
* Initial size of opened file for adjustDirSize call
*
* @var int
*/
protected $opened_size;
/**
* Cache containing stat-infos from previous url_stat calls AND dir_opendir calls
*
@ -326,6 +332,16 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
{
$this->stream_seek(0,SEEK_END);
}
// remember initial size and directory for adjustDirSize call in close
if (is_resource($this->opened_stream))
{
$this->opened_size = empty($stat) ? $stat['size'] : 0;
if (empty($dir_stat))
{
$dir_stat = $this->url_stat($dir,STREAM_URL_STAT_QUIET);
}
$this->opened_dir = $dir_stat['ino'];
}
if (!is_resource($this->opened_stream)) error_log(__METHOD__."($url,$mode,$options) NO stream, returning false!");
return is_resource($this->opened_stream);
@ -348,10 +364,11 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
if ($this->opened_mode != 'r')
{
$this->stream_seek(0,SEEK_END);
$size = $this->stream_tell();
// we need to update the mime-type, size and content (if STORE2DB)
$values = array(
'fs_size' => $this->stream_tell(),
'fs_size' => $size,
// todo: analyse the file for the mime-type
'fs_mime' => Api\MimeMagic::filename2mime($this->opened_path),
'fs_id' => $this->opened_fs_id,
@ -381,6 +398,11 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
error_log(__METHOD__."() execute() failed! errorInfo()=".array2string(self::$pdo->errorInfo()));
}
}
// adjust directory size, if changed
if ($ret && $size != $this->opened_size && $this->opened_dir)
{
$this->adjustDirSize($this->opened_dir, $size-$this->opened_size);
}
}
else
{
@ -584,10 +606,51 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
unset($stmt);
$stmt = self::$pdo->prepare('DELETE FROM '.self::PROPS_TABLE.' WHERE fs_id=?');
$stmt->execute(array($stat['ino']));
if ($stat['mime'] !== self::SYMLINK_MIME_TYPE)
{
$this->adjustDirSize($parent_stat['ino'], -$stat['size']);
}
}
return $ret;
}
/**
* Adjust directory sizes
*
* Adjustment is always relative, so concurrency does not matter.
* Adjustment is made to all parent directories too!
*
* @param int $fs_id fs_id of directory to adjust
* @param int $fs_size size adjustment
* @param bool $fs_id_is_dir=true false: $fs_id is the file causing the change (only adjust its directories)
*/
protected function adjustDirSize(int $fs_id, int $fs_size, bool $fs_id_is_dir=true)
{
if (!$fs_size) return; // nothing to do
static $stmt=null,$parent_stmt;
if (!isset($stmt))
{
$stmt = self::$pdo->prepare('UPDATE '.self::TABLE.' SET fs_size=fs_size+:fs_size WHERE fs_id=:fs_id');
$parent_stmt = self::$pdo->prepare('SELECT fs_dir FROM '.self::TABLE.' WHERE fs_id=:fs_id');
}
$max_depth = 100;
do
{
if ($fs_id_is_dir || $max_depth < 100)
{
$stmt->execute([
'fs_id' => $fs_id,
'fs_size' => $fs_size,
]);
}
$parent_stmt->execute(['fs_id' => $fs_id]);
}
while (($fs_id = $parent_stmt->fetchColumn()) && --$max_depth > 0);
}
/**
* This method is called in response to rename() calls on URL paths associated with the wrapper.
*
@ -667,6 +730,12 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
));
unset(self::$stat_cache[$path_to]);
}
if ($ok && $to_dir_stat['ino'] !== $from_dir_stat['ino'] && $new_mime !== self::SYMLINK_MIME_TYPE)
{
$this->adjustDirSize($from_dir_stat['ino'], -$from_stat['size']);
$this->adjustDirSize($to_dir_stat['ino'], $from_stat['size']);
}
return $ok;
}
@ -1390,7 +1459,7 @@ class StreamWrapper extends Api\Db\Pdo implements Vfs\StreamWrapperIface
'fs_modified' => self::_pdo_timestamp(time()),
'fs_creator' => Vfs::$user,
'fs_mime' => self::SYMLINK_MIME_TYPE,
'fs_size' => bytes($target),
'fs_size' => 0,
'fs_link' => $target,
));
}

53
api/src/Vfs/Sqlfs/Utils.php
View File

@ -103,7 +103,7 @@ class Utils extends StreamWrapper
}
/**
* Check and optionaly fix corruption in sqlfs
* Check and optionally fix corruption in sqlfs
*
* @param boolean $check_only =true
* @return array with messages / found problems
@ -132,6 +132,11 @@ class Utils extends StreamWrapper
{
if ($app_msgs) $msgs = array_merge($msgs, $app_msgs);
}
// also run quota recalc as fsck might have (re-)moved files
list($dirs, $iterations, $time) = Vfs\Sqlfs\Utils::quotaRecalc();
$msgs[] = lang("Recalculated %1 directories in %2 iterations and %3 seconds", $dirs, $iterations, number_format($time, 1));
return $msgs;
}
@ -501,6 +506,52 @@ class Utils extends StreamWrapper
}
return $msgs;
}
/**
* Recalculate directory sizes
*
* @return int[] directories recalculated, iterations necessary, time
*/
static function quotaRecalc()
{
if (!is_object(self::$pdo))
{
self::_pdo();
}
$start = microtime(true);
$table = self::TABLE;
self::$pdo->query("UPDATE $table SET fs_size=NULL WHERE fs_mime='".self::DIR_MIME_TYPE."' AND fs_active");
self::$pdo->query("UPDATE $table SET fs_size=0 WHERE fs_mime='".self::SYMLINK_MIME_TYPE."'");
$stmt = self::$pdo->prepare("SELECT $table.fs_id, SUM(child.fs_size) as total
FROM $table
LEFT JOIN $table child ON $table.fs_id=child.fs_dir AND child.fs_active
WHERE $table.fs_active AND $table.fs_mime='httpd/unix-directory' AND $table.fs_size IS NULL
GROUP BY $table.fs_id
HAVING COUNT(child.fs_id)=0 OR COUNT(CASE WHEN child.fs_size IS NULL THEN 1 ELSE NULL END)=0
ORDER BY $table.fs_id DESC
LIMIT 500");
$stmt->setFetchMode(PDO::FETCH_ASSOC);
$update = self::$pdo->prepare("UPDATE $table SET fs_size=:fs_size WHERE fs_id=:fs_id");
$iterations = $dirs = 0;
do
{
$rows = 0;
$stmt->execute();
foreach ($stmt as $row)
{
$update->execute([
'fs_size' => $row['total'] ?? 0,
'fs_id' => $row['fs_id'],
]);
++$rows;
}
$dirs += $rows;
}
while ($rows > 0 && ++$iterations < 100);
return [$dirs, $iterations, microtime(true)-$start];
}
}
// fsck testcode, if this file is called via it's URL (you need to uncomment it!)

11
filemanager/inc/class.filemanager_admin.inc.php
View File

@ -29,6 +29,7 @@ class filemanager_admin extends filemanager_ui
public $public_functions = array(
'index' => true,
'fsck' => true,
'quotaRecalc' => true,
);
/**
@ -365,4 +366,14 @@ class filemanager_admin extends filemanager_ui
$GLOBALS['egw']->framework->render($content, lang('Admin').' - '.lang('Check virtual filesystem'), true);
}
/**
* Recalculate directory sizes
*/
function quotaRecalc()
{
list($dirs, $iterations, $time) = Vfs\Sqlfs\Utils::quotaRecalc();
echo lang("Recalculated %1 directories in %2 iterations and %3 seconds", $dirs, $iterations, number_format($time, 1))."\n";
}
}

1
filemanager/inc/class.filemanager_hooks.inc.php
View File

@ -97,6 +97,7 @@ class filemanager_hooks
//'Site Configuration' => Egw::link('/index.php','menuaction=admin.admin_config.index&appname='.self::$appname.'&ajax=true'),
'Custom fields' => Egw::link('/index.php','menuaction=admin.admin_customfields.index&appname='.self::$appname.'&ajax=true'),
'Check virtual filesystem' => Egw::link('/index.php','menuaction=filemanager.filemanager_admin.fsck'),
'Recalculate quota' => Egw::link('/index.php','menuaction=filemanager.filemanager_admin.quotaRecalc'),
'VFS mounts and versioning' => Egw::link('/index.php', 'menuaction=filemanager.filemanager_admin.index&ajax=true'),
);
if ($location == 'admin')