extern
/
egroupware_official
mirror of https://github.com/EGroupware/egroupware.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP S3 stream-wrapper: refactored DB backup in order to (optionally) use S3

This commit is contained in:
ralf 2023-10-24 22:22:16 +03:00
parent effa3686c8
commit b45b3c35df
3 changed files with 187 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
admin/inc/class.admin_db_backup.inc.php
View File

@ -3,36 +3,55 @@
* EGroupware - Admin - DB backup and restore
*
* @link http://www.egroupware.org
* @author Ralf Becker <RalfBecker@outdoor-training.de>
* @author Ralf Becker <rb@egroupware.org>
* @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
* @package admin
* @version $Id$
*/
use EGroupware\Api;
use EGroupware\Stylite\Vfs\S3;
class admin_db_backup
{
var $public_functions = array(
/**
* @var true[]
*/
public $public_functions = array(
'index' => true,
);
var $db_backup;
/**
* @var Api\Db\Backup
*/
protected $db_backup;
/**
* Method for sheduled backups, called via asynservice
* Method for scheduled backups, called via asynservice
*/
function do_backup()
{
$this->db_backup = new Api\Db\Backup();
if (class_exists(S3\Backup::class) && S3\Backup::available())
{
$this->db_backup = new S3\Backup();
}
else
{
$this->db_backup = new Api\Db\Backup();
}
if (($f = $this->db_backup->fopen_backup()))
{
try {
$f = $this->db_backup->fopen_backup();
$this->db_backup->backup($f);
if(is_resource($f))
if (is_resource($f))
{
fclose($f);
}
/* Remove old backups. */
$this->db_backup->housekeeping();
}
catch (\Exception $e) {
// log error
_egw_log_exception($e);
}
}
/**
@ -50,4 +69,4 @@ class admin_db_backup
unset($tpl_root, $self);
echo $GLOBALS['egw']->framework->footer();
}
}
}

152
api/src/Db/Backup.php
View File

@ -31,25 +31,25 @@ class Backup
*
* @var Api\Db\Schema
*/
var $schema_proc;
protected $schema_proc;
/**
* Reference to ADOdb (connection) object
*
* @var ADOConnection
*/
var $adodb;
protected $adodb;
/**
* DB schemas, as array tablename => schema
*
* @var array
*/
var $schemas = array();
protected $schemas = array();
/**
* Tables to exclude from the backup: sessions, diverse caches which get automatic rebuild
*
* @var array
*/
var $exclude_tables = array(
public $exclude_tables = array(
'egw_sessions','egw_app_sessions','phpgw_sessions','phpgw_app_sessions', // eGW's session-tables
'phpgw_anglemail', // email's cache
'egw_felamimail_cache','egw_felamimail_folderstatus','phpgw_felamimail_cache','phpgw_felamimail_folderstatus', // felamimail's cache
@ -60,13 +60,13 @@ class Backup
*
* @var string|boolean
*/
var $system_tables = false;
public $system_tables = false;
/**
* Regular expression to identify eGW tables => if set only they are used
*
* @var string|boolean
*/
var $egw_tables = false;
public $egw_tables = false;
/**
* Regular expression to identify a Guacamole table OR view
*/
@ -76,25 +76,25 @@ class Backup
*
* @var string
*/
var $backup_dir;
public $backup_dir;
/**
* Minimum number of backup files to keep. Zero for: Disable cleanup.
*
* @var integer
*/
var $backup_mincount;
public $backup_mincount;
/**
* Backup Files config value, will be overwritten by the availability of the ZibArchive libraries
*
* @var boolean
*/
var $backup_files = false ;
public $backup_files = false ;
/**
* Reference to schema_proc's Api\Db object
*
* @var Api\Db
*/
var $db;
protected $db;
/**
* Constructor
@ -179,65 +179,66 @@ class Backup
/**
* Opens the backup-file using the highest available compression
*
* @param $name =false string/boolean filename to use, or false for the default one
* @param $reading =false opening for reading ('rb') or writing ('wb')
* @return string/resource/zip error-msg of file-handle
* @param ?string $name =false string/boolean filename to use, or false for the default one
* @param bool $reading =false opening for reading ('rb') or writing ('wb')
* @param bool $un_compress true: opening file with (un)compression wrapper, false: opening as it is for download
* @return resource file-handle
* @throws \Exception on error
*/
function fopen_backup($name=false,$reading=false)
public function fopen_backup(string $name=null, bool $reading=false, bool $un_compress=true)
{
//echo "function fopen_backup($name,$reading)<br>"; // !
if ($name)
{
$name = $this->backup_dir.'/'.basename($name);
}
if (!$name)
{
//echo '-> !$name<br>'; // !
if (empty($this->backup_dir) || !is_writable($this->backup_dir))
{
$this->log($name, $reading, null, lang("backupdir '%1' is not writeable by the webserver", $this->backup_dir));
return lang("backupdir '%1' is not writeable by the webserver",$this->backup_dir);
throw new Exception(lang("backupdir '%1' is not writeable by the webserver", $this->backup_dir));
}
$name = $this->backup_dir.'/db_backup-'.date('YmdHi');
}
else // remove the extension, to use the correct wrapper based on the extension
// remove the extension, to use the correct wrapper based on the extension
elseif ($un_compress)
{
//echo '-> else<br>'; // !
$name = preg_replace('/\.(bz2|gz)$/i','',$name);
}
$mode = $reading ? 'rb' : 'wb';
list( , $type) = explode('.', basename($name));
if($type == 'zip' && $reading && $this->backup_files)
$lang_mode = $reading ? lang("reading") : lang("writing");
[, $type] = explode('.', basename($name));
if($un_compress && $type == 'zip' && $reading && $this->backup_files)
{
//echo '-> $type == "zip" && $reading<br>'; // !
if(!class_exists('ZipArchive', false))
{
$this->backup_files = false;
$this->log($name, $reading, null, lang("Cant open %1, needs ZipArchive", $name));
return lang("Cant open %1, needs ZipArchive", $name)."<br>\n";
throw new \Exception(lang("Cant open %1, needs ZipArchive", $name));
}
if(!($f = fopen($path=$name, $mode)))
{
//echo ' -> !($f = fopen($name, $mode))<br>'; // !
$lang_mode = $reading ? lang("reading") : lang("writing");
$this->log($name, $reading, null, lang("Cant open '%1' for %2", $name, $lang_mode));
return lang("Cant open '%1' for %2", $name, $lang_mode)."<br>";
throw new \Exception(lang("Cant open '%1' for %2", $name, $lang_mode));
}
}
elseif (class_exists('ZipArchive', false) && !$reading && $this->backup_files)
elseif ($un_compress && class_exists('ZipArchive', false) && !$reading && $this->backup_files)
{
//echo '-> (new ZipArchive) != NULL && !$reading; '.$name.'<br>'; // !
if (!($f = fopen($path=$name, $mode)))
{
$lang_mode = $reading ? lang("reading") : lang("writing");
$this->log($name, $reading, null, lang("Cant open '%1' for %2", $name, $lang_mode));
return lang("Cant open '%1' for %2", $name, $lang_mode)."<br>";
throw new \Exception(lang("Cant open '%1' for %2", $name, $lang_mode));
}
}
elseif (!($f = fopen('compress.bzip2://'.($path=$name.'.bz2'), $mode)) &&
!($f = fopen('compress.zlib://'.($path=$name.'.gz'),$mode)) &&
!($f = fopen($path=$name,$mode))
)
elseif (!($un_compress && (
($f = fopen('compress.bzip2://'.($path=$name.'.bz2'), $mode)) ||
($f = fopen('compress.zlib://'.($path=$name.'.gz'),$mode)))) &&
!($f = fopen($path=$name,$mode)))
{
$lang_mode = $reading ? lang("reading") : lang("writing");
$this->log($name, $reading, null, lang("Cant open '%1' for %2", $name, $lang_mode));
return lang("Cant open '%1' for %2", $name, $lang_mode)."<br>";
throw new \Exception(lang("Cant open '%1' for %2", $name, $lang_mode));
}
// Log start of backup/restore
@ -1228,6 +1229,89 @@ class Backup
error_log("Could NOT open ".self::LOG_FILE.': '.$msg);
}
}
/**
* Move uploaded file to backup-directory
*
* @param array $file values for keys "tmp_name", "name", "size"
* @return ?string success message or null on error
*/
public function upload(array $file) : ?string
{
if (move_uploaded_file($file['tmp_name'], $filename = $this->backup_dir . '/' . basename($file['name'])))
{
$msg = lang("succesfully uploaded file %1", $filename . ', ' .
sprintf('%3.1f MB (%d)', $file['size'] / (1024 * 1024), $file['size'])) .
', md5=' . md5_file($file['tmp_name']) . ', sha1=' . sha1_file($file['tmp_name']);
$this->log($filename, $msg);
}
return $msg ?? null;
}
/**
* Delete a backup
*
* @param string $name filename
* @return string|null success message or null on error
*/
public function delete(string $name) : ?string
{
if (unlink($file = $this->backup_dir.'/'.basename($name))) // basename to not allow to change the dir
{
$this->log($file, $msg = lang("backup '%1' deleted", $file));
}
return $msg ?? null;
}
/**
* Rename a backup
* @param string $file
* @param string $new_name
* @return string|null success message or null on error
*/
public function rename(string $file, string $new_name) : ?string
{
[$ending] = array_reverse(explode('.', $file));
[$new_ending, $has_ending] = array_reverse(explode('.', $new_name));
if (!$has_ending || $new_ending != $ending)
{
$new_name .= '.' . $ending;
}
$file = $this->backup_dir.'/'.basename($file); // basename to not allow to change the dir
$ext = preg_match('/(\.gz|\.bz2)+$/i',$file,$matches) ? $matches[1] : '';
$new_file = $this->backup_dir.'/'.preg_replace('/(\.gz|\.bz2)+$/i','',basename($new_name)).$ext;
if (rename($file, $new_file))
{
$this->log($new_file, $msg=lang("backup '%1' renamed to '%2'", basename($file), basename($new_file)));
}
return $msg ?? null;
}
/**
* List available backups for restore
*
* @return array filename => [$ctime, $size] pairs
*/
public function index() : array
{
$files = [];
if (($handle = opendir($this->backup_dir)))
{
while(($file = readdir($handle)))
{
if ($file != '.' && $file != '..')
{
$files[$file] = [
'ctime' => filectime($this->backup_dir.'/'.$file),
'size' => filesize($this->backup_dir.'/'.$file)
];
}
}
closedir($handle);
}
arsort($files);
return $files;
}
}
/*

105
setup/db_backup.php
View File

@ -13,6 +13,7 @@ use EGroupware\Api;
use EGroupware\Api\Framework;
use EGroupware\Api\Egw;
use EGroupware\Api\Vfs;
use EGroupware\Stylite\Vfs\S3;
if (!is_object(@$GLOBALS['egw'])) // called from outside EGw ==> setup
{
@ -34,23 +35,31 @@ if (!is_object(@$GLOBALS['egw'])) // called from outside EGw ==> setup
$self = 'db_backup.php';
$is_setup = true;
}
$db_backup = new Api\Db\Backup();
if (class_exists(S3\Backup::class) && S3\Backup::available())
{
$db_backup = new S3\Backup();
}
else
{
$db_backup = new Api\Db\Backup();
}
$asyncservice = new Api\Asyncservice();
// download a backup, has to be before any output !!!
if (!empty($_POST['download']))
{
$file = key($_POST['download']);
$file = $db_backup->backup_dir.'/'.basename($file); // basename to not allow to change the dir
$filename = $db_backup->backup_dir.'/'.key($_POST['download']);
$file = $db_backup->fopen_backup($filename, true, false);
// FIRST: switch off zlib.output_compression, as this would limit downloads in size to memory_limit
ini_set('zlib.output_compression',0);
// SECOND: end all active output buffering
while(ob_end_clean()) {}
Api\Header\Content::type(basename($file));
readfile($file);
$db_backup->log($file, 'Downloaded');
Api\Header\Content::type(basename($filename));
fpassthru($file);
fclose($file);
$db_backup->log($filename, 'Downloaded');
exit;
}
$setup_tpl = new Framework\Template($tpl_root);
@ -100,7 +109,7 @@ if (!empty($_POST['save_backup_settings']))
$minCount = 0;
$setup_tpl->set_var('error_msg',htmlspecialchars(lang("'%1' must be integer", lang("backup min count"))));
}
$db_backup->saveConfig($minCount,$is_setup ? (boolean)$filesBackup : null);
$db_backup->saveConfig($minCount,!empty($is_setup) ? $filesBackup : null);
if (is_int($minCount) && $minCount > 0)
{
@ -126,12 +135,14 @@ if (!empty($_POST['mount']))
// create a backup now
if (!empty($_POST['backup']))
{
if (is_resource($f = $db_backup->fopen_backup()))
{
try {
$f = $db_backup->fopen_backup();
$starttime = microtime(true);
$db_backup->backup($f);
if(is_resource($f))
{
fclose($f);
}
$setup_tpl->set_var('error_msg', lang('backup finished').': '. number_format(microtime(true)-$starttime, 1).'s');
/* Remove old backups. */
@ -142,9 +153,8 @@ if (!empty($_POST['backup']))
echo '<div align="center">'.lang('entry has been deleted sucessfully').': '.$file."</div>\n";
}
}
else
{
$setup_tpl->set_var('error_msg',$f);
catch (\Exception $e) {
$setup_tpl->set_var('error_msg', $e->getMessage());
}
}
$setup_tpl->set_var('backup_now_button','<input type="submit" name="backup" title="'.
@ -161,43 +171,20 @@ $setup_tpl->set_var('backup_save_settings','<input type="submit" name="save_back
$setup_tpl->set_var('backup_mount','<input type="submit" name="mount" value="'.htmlspecialchars(lang('Mount backup directory to %1','/backup')).'" />');
if (!empty($_POST['upload']) && is_array($_FILES['uploaded']) && !$_FILES['uploaded']['error'] &&
is_uploaded_file($_FILES['uploaded']['tmp_name']))
is_uploaded_file($_FILES['uploaded']['tmp_name']) && ($msg = $db_backup->upload($_FILES['uploaded'])))
{
move_uploaded_file($_FILES['uploaded']['tmp_name'], $filename=$db_backup->backup_dir.'/'.$_FILES['uploaded']['name']);
$md5 = ', md5='.md5_file($filename).', sha1='.sha1_file($filename);
$setup_tpl->set_var('error_msg', ($msg=lang("succesfully uploaded file %1", $filename.', '.
sprintf('%3.1f MB (%d)',$_FILES['uploaded']['size']/(1024*1024),$_FILES['uploaded']['size']))).$md5);
$db_backup->log($filename, $msg);
$setup_tpl->set_var('error_msg', $msg);
}
// delete a backup
if (!empty($_POST['delete']))
if (!empty($_POST['delete']) && ($msg = $db_backup->delete(key($_POST['delete']))))
{
$file = $db_backup->backup_dir.'/'.basename(key($_POST['delete'])); // basename to not allow to change the dir
$db_backup->log($file, lang("backup '%1' deleted", $file));
if (unlink($file)) $setup_tpl->set_var('error_msg',lang("backup '%1' deleted",$file));
$setup_tpl->set_var('error_msg', $msg);
}
// rename a backup
if (!empty($_POST['rename']))
if (!empty($file=$_POST['rename']) && !empty($_POST['new_name'][$file]) &&
($msg = $db_backup->rename($file, $_POST['new_name'][$file])))
{
$file = key($_POST['rename']);
$new_name = $_POST['new_name'][$file];
if (!empty($new_name))
{
list($ending) = array_reverse(explode('.', $file));
list($new_ending, $has_ending) = array_reverse(explode('.', $new_name));
if(!$has_ending || $new_ending != $ending) $new_name .= '.'.$ending;
$file = $db_backup->backup_dir.'/'.basename($file); // basename to not allow to change the dir
$ext = preg_match('/(\.gz|\.bz2)+$/i',$file,$matches) ? $matches[1] : '';
$new_file = $db_backup->backup_dir.'/'.preg_replace('/(\.gz|\.bz2)+$/i','',basename($new_name)).$ext;
if (rename($file,$new_file))
{
$setup_tpl->set_var('error_msg',lang("backup '%1' renamed to '%2'",basename($file),basename($new_file)));
$db_backup->log($new_file, lang("backup '%1' renamed to '%2'",basename($file),basename($new_file)));
}
}
$setup_tpl->set_var('error_msg', $msg);
}
// restore a backup
if (!empty($_POST['restore']))
@ -205,12 +192,12 @@ if (!empty($_POST['restore']))
$file = key($_POST['restore']);
$file = $db_backup->backup_dir.'/'.basename($file); // basename to not allow to change the dir
if (is_resource($f = $db_backup->fopen_backup($file,true)))
{
try {
$f = $db_backup->fopen_backup($file,true);
$start = time();
$db_backup->restore($f, true, $file); // allways convert to current system charset on restore
$db_backup->restore($f, true, $file); // always convert to current system charset on restore
$setup_tpl->set_var('error_msg',lang("backup '%1' restored",$file).' ('.(time()-$start).' s)');
if ($run_in_egw)
if (isset($run_in_egw))
{
// updating the backup
$cmd = new setup_cmd_update($GLOBALS['egw']->session->account_domain,
@ -220,9 +207,9 @@ if (!empty($_POST['restore']))
echo '<h3>'.lang('You should %1log out%2 and in again, to update your current session!','<a href="'.Egw::link('/logout.php').'" target="_parent">','</a>')."</h3>\n";
}
}
else
catch (\Exception $e)
{
$setup_tpl->set_var('error_msg',$f);
$setup_tpl->set_var('error_msg', $e->getMessage());
}
}
// create a new scheduled backup
@ -256,28 +243,16 @@ $setup_tpl->set_var('next_run','&nbsp;');
$setup_tpl->set_var('actions','<input type="submit" name="schedule" value="'.htmlspecialchars(lang('schedule')).'" />');
$setup_tpl->parse('schedule_rows','schedule_row',true);
// listing the availible backup sets
// listing the available backups
$setup_tpl->set_var('backup_dir',$db_backup->backup_dir);
$setup_tpl->set_var('set_rows','');
$handle = @opendir($db_backup->backup_dir);
$files = array();
while($handle && ($file = readdir($handle)))
{
if ($file != '.' && $file != '..')
{
$files[$file] = filectime($db_backup->backup_dir.'/'.$file);
}
}
if ($handle) closedir($handle);
arsort($files);
foreach($files as $file => $ctime)
foreach($db_backup->index() as $file => $attrs)
{
$size = filesize($db_backup->backup_dir.'/'.$file);
$setup_tpl->set_var(array(
'filename' => $file,
'mod' => date('Y-m-d H:i',$ctime),
'size' => sprintf('%3.1f MB (%d)',$size/(1024*1024),$size),
'mod' => date('Y-m-d H:i', $attrs['ctime']),
'size' => sprintf('%3.1f MB (%d)',$attrs['size']/(1024*1024), $attrs['size']),
'actions' => '<input type="submit" name="download['.$file.']" value="'.htmlspecialchars(lang('download')).'" />&nbsp;'."\n".
($file === Api\Db\Backup::LOG_FILE ? '' :
'<input type="submit" name="delete['.$file.']" value="'.htmlspecialchars(lang('delete')).'" onclick="return confirm(\''.
@ -315,7 +290,7 @@ $setup_tpl->set_var(array(
$setup_tpl->set_var('self',$self);
$setup_tpl->pparse('out','T_db_backup');
if ($run_in_egw)
if (isset($run_in_egw))
{
echo $GLOBALS['egw']->framework->footer();
}