extern/egroupware_official
1
0
Fork 0
mirror of https://github.com/EGroupware/egroupware.git synced 2024-11-24 17:04:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix SQLfs fsck to query errors in chunks of 500, in case there are a huge numbers of them

Browse Source
This commit is contained in:
Ralf Becker 2021-12-14 21:24:31 +02:00
parent 1448581403
commit 9439c2d799
1 changed files with 99 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
api/src/Vfs/Sqlfs/Utils.php
View File

@ -14,6 +14,7 @@ namespace EGroupware\Api\Vfs\Sqlfs;
use EGroupware\Api\Vfs; use EGroupware\Api\Vfs;
use EGroupware\Api; use EGroupware\Api;
use PDO, PDOException;
/** /**
* sqlfs stream wrapper utilities: migration db-fs, fsck * sqlfs stream wrapper utilities: migration db-fs, fsck
@ -33,7 +34,7 @@ class Utils extends StreamWrapper
} }
$query = 'SELECT fs_id,fs_name,fs_size,fs_content'. $query = 'SELECT fs_id,fs_name,fs_size,fs_content'.
' FROM '.self::TABLE.' WHERE fs_content IS NOT NULL'. ' FROM '.self::TABLE.' WHERE fs_content IS NOT NULL'.
' ORDER BY fs_id LIMIT 5 OFFSET :offset'; ' ORDER BY fs_id LIMIT 500 OFFSET :offset';
$fs_id = $fs_name = $fs_size = $fs_content = null; $fs_id = $fs_name = $fs_size = $fs_content = null;
$n = 0; $n = 0;
@ -41,8 +42,8 @@ class Utils extends StreamWrapper
$stmt->bindColumn(1,$fs_id); $stmt->bindColumn(1,$fs_id);
$stmt->bindColumn(2,$fs_name); $stmt->bindColumn(2,$fs_name);
$stmt->bindColumn(3,$fs_size); $stmt->bindColumn(3,$fs_size);
$stmt->bindColumn(4,$fs_content,\PDO::PARAM_LOB); $stmt->bindColumn(4,$fs_content,PDO::PARAM_LOB);
$stmt->bindValue(':offset', $n, \PDO::PARAM_INT); $stmt->bindValue(':offset', $n, PDO::PARAM_INT);
while ($stmt->execute()) // && $stmt->rowCount() does not work for all dbs :( while ($stmt->execute()) // && $stmt->rowCount() does not work for all dbs :(
{ {
@ -50,7 +51,7 @@ class Utils extends StreamWrapper
foreach($stmt as $row) foreach($stmt as $row)
{ {
// hack to work around a current php bug (http://bugs.php.net/bug.php?id=40913) // hack to work around a current php bug (http://bugs.php.net/bug.php?id=40913)
// PDOStatement::bindColumn(,,\PDO::PARAM_LOB) is not working for MySQL, content is returned as string :-( // PDOStatement::bindColumn(,,PDO::PARAM_LOB) is not working for MySQL, content is returned as string :-(
if (is_string($fs_content)) if (is_string($fs_content))
{ {
$content = fopen('php://temp', 'wb'); $content = fopen('php://temp', 'wb');
@ -87,7 +88,7 @@ class Utils extends StreamWrapper
} }
if (!$n || $n == $start) break; // just in case nothing is found, statement will execute just fine if (!$n || $n == $start) break; // just in case nothing is found, statement will execute just fine
$stmt->bindValue(':offset', $n, \PDO::PARAM_INT); $stmt->bindValue(':offset', $n, PDO::PARAM_INT);
} }
unset($row); // not used, as we access bound variables unset($row); // not used, as we access bound variables
unset($stmt); unset($stmt);
@ -180,7 +181,7 @@ class Utils extends StreamWrapper
'fs_creator' => 0, 'fs_creator' => 0,
)); ));
} }
catch (\PDOException $e) catch (PDOException $e)
{ {
$ok = false; $ok = false;
unset($e); // ignore exception unset($e); // ignore exception
@ -194,7 +195,7 @@ class Utils extends StreamWrapper
try { try {
$ok = $delete_stmt->execute(array('fs_id' => $id)) && $stmt->execute($data); $ok = $delete_stmt->execute(array('fs_id' => $id)) && $stmt->execute($data);
} }
catch (\PDOException $e) catch (PDOException $e)
{ {
unset($e); // ignore exception unset($e); // ignore exception
} }
@ -248,37 +249,50 @@ class Utils extends StreamWrapper
{ {
$stmt = null; $stmt = null;
$msgs = array(); $msgs = array();
foreach(self::$pdo->query('SELECT fs_id FROM '.self::TABLE. $limit = 500;
" WHERE fs_mime!='httpd/unix-directory' AND fs_content IS NULL AND fs_link IS NULL") as $row) $offset = 0;
{ $select_stmt = self::$pdo->prepare('SELECT fs_id FROM '.self::TABLE.
if (!file_exists($phy_path=self::_fs_path($row['fs_id']))) " WHERE fs_mime!='httpd/unix-directory' AND fs_content IS NULL AND fs_link IS NULL LIMIT $limit OFFSET :offset");
$select_stmt->setFetchMode(PDO::FETCH_ASSOC);
do {
$num = 0;
$select_stmt->bindValue(':offset', $offset, PDO::PARAM_INT);
foreach($select_stmt->execute() as $row)
{ {
$path = self::id2path($row['fs_id']); ++$num;
if ($check_only) if (!file_exists($phy_path=self::_fs_path($row['fs_id'])))
{ {
$msgs[] = lang('File %1 has no content in physical filesystem %2!', $path = self::id2path($row['fs_id']);
$path.' (#'.$row['fs_id'].')',$phy_path); if ($check_only)
}
else
{
if (!isset($stmt))
{ {
$stmt = self::$pdo->prepare('DELETE FROM '.self::TABLE.' WHERE fs_id=:fs_id'); ++$offset;
$stmt_props = self::$pdo->prepare('DELETE FROM '.self::PROPS_TABLE.' WHERE fs_id=:fs_id'); $msgs[] = lang('File %1 has no content in physical filesystem %2!',
} $path.' (#'.$row['fs_id'].')',$phy_path);
if ($stmt->execute(array('fs_id' => $row['fs_id'])) &&
$stmt_props->execute(array('fs_id' => $row['fs_id'])))
{
$msgs[] = lang('File %1 has no content in physical filesystem %2 --> file removed!',$path,$phy_path);
} }
else else
{ {
$msgs[] = lang('File %1 has no content in physical filesystem %2 --> failed to remove file!', if (!isset($stmt))
$path.' (#'.$row['fs_id'].')',$phy_path); {
$stmt = self::$pdo->prepare('DELETE FROM '.self::TABLE.' WHERE fs_id=:fs_id');
$stmt_props = self::$pdo->prepare('DELETE FROM '.self::PROPS_TABLE.' WHERE fs_id=:fs_id');
}
if ($stmt->execute(array('fs_id' => $row['fs_id'])) &&
$stmt_props->execute(array('fs_id' => $row['fs_id'])))
{
$msgs[] = lang('File %1 has no content in physical filesystem %2 --> file removed!',$path,$phy_path);
}
else
{
++$offset;
$msgs[] = lang('File %1 has no content in physical filesystem %2 --> failed to remove file!',
$path.' (#'.$row['fs_id'].')',$phy_path);
}
} }
} }
} }
} }
while ($num >= $limit);
if ($check_only && $msgs) if ($check_only && $msgs)
{ {
$msgs[] = lang('Files without content in physical filesystem will be removed.'); $msgs[] = lang('Files without content in physical filesystem will be removed.');
@ -309,55 +323,69 @@ class Utils extends StreamWrapper
$lostnfound = null; $lostnfound = null;
$msgs = array(); $msgs = array();
$sqlfs = new StreamWrapper(); $sqlfs = new StreamWrapper();
foreach(self::$pdo->query('SELECT fs.* FROM '.self::TABLE.' fs'. $limit = 500;
' LEFT JOIN '.self::TABLE.' dir ON dir.fs_id=fs.fs_dir'. $offset = 0;
' WHERE fs.fs_id > 1 AND dir.fs_id IS NULL') as $row) $stmt = self::$pdo->prepare('SELECT fs.* FROM ' . self::TABLE . ' fs' .
' LEFT JOIN ' . self::TABLE . ' dir ON dir.fs_id=fs.fs_dir' .
" WHERE fs.fs_id > 1 AND dir.fs_id IS NULL LIMIT $limit OFFSET :offset");
$stmt->setFetchMode(PDO::FETCH_ASSOC);
do
{ {
if ($check_only) $num = 0;
$stmt->bindValue(':offset', $offset, PDO::PARAM_INT);
foreach ($stmt->execute() as $row)
{ {
$msgs[] = lang('Found unconnected %1 %2!', ++$num;
Api\MimeMagic::mime2label($row['fs_mime']), if ($check_only)
Vfs::decodePath($row['fs_name']).' (#'.$row['fs_id'].')');
continue;
}
if (!isset($lostnfound))
{
// check if we already have /lost+found, create it if not
if (!($lostnfound = $sqlfs->url_stat(self::LOST_N_FOUND, STREAM_URL_STAT_QUIET)))
{ {
Vfs::$is_root = true; ++$offset;
if (!$sqlfs->mkdir(self::LOST_N_FOUND, self::LOST_N_FOUND_MOD, 0) || $msgs[] = lang('Found unconnected %1 %2!',
!(!($admins = $GLOBALS['egw']->accounts->name2id(self::LOST_N_FOUND_GRP)) || Api\MimeMagic::mime2label($row['fs_mime']),
self::chgrp(self::LOST_N_FOUND, $admins) && self::chmod(self::LOST_N_FOUND,self::LOST_N_FOUND_MOD)) || Vfs::decodePath($row['fs_name']) . ' (#' . $row['fs_id'] . ')');
!($lostnfound = $sqlfs->url_stat(self::LOST_N_FOUND, STREAM_URL_STAT_QUIET))) continue;
{ }
$msgs[] = lang("Can't create directory %1 to connect found unconnected nodes to it!",self::LOST_N_FOUND); if (!isset($lostnfound))
} {
else // check if we already have /lost+found, create it if not
{ if (!($lostnfound = $sqlfs->url_stat(self::LOST_N_FOUND, STREAM_URL_STAT_QUIET)))
$msgs[] = lang('Successful created new directory %1 for unconnected nods.',self::LOST_N_FOUND); {
} Vfs::$is_root = true;
Vfs::$is_root = false; if (!$sqlfs->mkdir(self::LOST_N_FOUND, self::LOST_N_FOUND_MOD, 0) ||
if (!$lostnfound) break; !(!($admins = $GLOBALS['egw']->accounts->name2id(self::LOST_N_FOUND_GRP)) ||
self::chgrp(self::LOST_N_FOUND, $admins) && self::chmod(self::LOST_N_FOUND, self::LOST_N_FOUND_MOD)) ||
!($lostnfound = $sqlfs->url_stat(self::LOST_N_FOUND, STREAM_URL_STAT_QUIET)))
{
$msgs[] = lang("Can't create directory %1 to connect found unconnected nodes to it!", self::LOST_N_FOUND);
}
else
{
$msgs[] = lang('Successful created new directory %1 for unconnected nods.', self::LOST_N_FOUND);
}
Vfs::$is_root = false;
if (!$lostnfound) break;
}
$stmt = self::$pdo->prepare('UPDATE ' . self::TABLE . ' SET fs_dir=:fs_dir WHERE fs_id=:fs_id');
}
if ($stmt->execute(array(
'fs_dir' => $lostnfound['ino'],
'fs_id' => $row['fs_id'],
)))
{
$msgs[] = lang('Moved unconnected %1 %2 to %3.',
Api\MimeMagic::mime2label($row['fs_mime']),
Vfs::decodePath($row['fs_name']) . ' (#' . $row['fs_id'] . ')',
self::LOST_N_FOUND);
}
else
{
++$offset;
$msgs[] = lang('Failed to move unconnected %1 %2 to %3!',
Api\MimeMagic::mime2label($row['fs_mime']), Vfs::decodePath($row['fs_name']), self::LOST_N_FOUND);
} }
$stmt = self::$pdo->prepare('UPDATE '.self::TABLE.' SET fs_dir=:fs_dir WHERE fs_id=:fs_id');
}
if ($stmt->execute(array(
'fs_dir' => $lostnfound['ino'],
'fs_id' => $row['fs_id'],
)))
{
$msgs[] = lang('Moved unconnected %1 %2 to %3.',
Api\MimeMagic::mime2label($row['fs_mime']),
Vfs::decodePath($row['fs_name']).' (#'.$row['fs_id'].')',
self::LOST_N_FOUND);
}
else
{
$msgs[] = lang('Failed to move unconnected %1 %2 to %3!',
Api\MimeMagic::mime2label($row['fs_mime']), Vfs::decodePath($row['fs_name']), self::LOST_N_FOUND);
} }
} }
while ($num >= $limit);
if ($check_only && $msgs) if ($check_only && $msgs)
{ {
$msgs[] = lang('Unconnected nodes will be moved to %1.',self::LOST_N_FOUND); $msgs[] = lang('Unconnected nodes will be moved to %1.',self::LOST_N_FOUND);