WIP S3 stream-wrapper: fixes to backup and restore binary AES key column egw_sqlfs.fs_aes_key

This commit is contained in:
ralf 2023-10-30 13:30:56 +02:00
parent d15c63198c
commit 6ee162ccd5
9 changed files with 270 additions and 228 deletions

View File

@ -322,8 +322,10 @@ class Config
/**
* Initialise class: reference to db and self::$configs cache
*
* @param bool $force_reload
*/
public static function init_static()
public static function init_static(bool $force_reload=false)
{
// we use a reference here (no clone), as we no longer use Db::row() or Db::next_record()!
if (isset($GLOBALS['egw_setup']) && $GLOBALS['egw_setup']->db instanceof Db)
@ -335,7 +337,7 @@ class Config
self::$db = $GLOBALS['egw']->db;
}
// if item is not cached or cache is not looking alright --> query config from database
if (!(self::$configs = Cache::getInstance(__CLASS__, 'configs')) || !is_array(self::$configs['phpgwapi']))
if ($force_reload || !(self::$configs = Cache::getInstance(__CLASS__, 'configs')) || !is_array(self::$configs['phpgwapi']))
{
self::$configs = array();
foreach(self::$db->select(self::TABLE,'*',false,__LINE__,__FILE__) as $row)

View File

@ -1631,8 +1631,8 @@ class Db
// MySQL and MariaDB not 10.1 need 4-byte utf8 chars replaced with our default utf8 charset
// (MariaDB 10.1 does the replacement automatic, 10.0 cuts everything off behind and MySQL gives an error)
// (MariaDB 10.3 gives an error too: Incorrect string value: '\xF0\x9F\x98\x8A\x0AW...')
// Changing charset to utf8mb4 requires schema update, shortening of some indexes and probably have negative impact on performace!
if (isset($value) && substr($this->Type, 0, 5) === 'mysql')
// Changing charset to utf8mb4 requires schema update, shortening of some indexes and probably have negative impact on performance!
if (isset($value) && substr($this->Type, 0, 5) === 'mysql' && !in_array($type, ['binary', 'blob']))
{
$value = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $value);
}

View File

@ -127,7 +127,7 @@ class Backup
}
if (!($this->backup_dir = $this->db->query("SELECT config_value FROM {$GLOBALS['egw_setup']->config_table} WHERE config_app='phpgwapi' AND config_name='backup_dir'",__LINE__,__FILE__)->fetchColumn()))
{
$this->backup_dir = $this->files_dir.'/db_backup';
$this->backup_dir = dirname($this->files_dir).'/backup';
}
$this->charset = $this->db->query("SELECT config_value FROM {$GLOBALS['egw_setup']->config_table} WHERE config_app='phpgwapi' AND config_name='system_charset'",__LINE__,__FILE__)->fetchColumn();
$this->api_version = $this->db->select($GLOBALS['egw_setup']->applications_table,'app_version',array('app_name'=>array('api','phpgwapi')),
@ -143,7 +143,7 @@ class Backup
{
if (!($this->backup_dir = $GLOBALS['egw_info']['server']['backup_dir']))
{
$this->backup_dir = $GLOBALS['egw_info']['server']['files_dir'].'/db_backup';
$this->backup_dir = dirname($GLOBALS['egw_info']['server']['files_dir']).'/backup';
}
$this->files_dir = $GLOBALS['egw_info']['server']['files_dir'];
$this->backup_mincount = $GLOBALS['egw_info']['server']['backup_mincount'];
@ -335,7 +335,7 @@ class Backup
);
/**
* Backup all data in the form of a (compressed) csv file
* Restore a backup
*
* @param resource $f file opened with fopen for reading
* @param boolean $convert_to_system_charset =true obsolet, it's now allways done
@ -394,7 +394,7 @@ class Backup
// it could be an old backup
list( , $type) = explode('.', basename($filename));
$dir = $this->files_dir; // $GLOBALS['egw_info']['server']['files_dir'];
// we may have to clean up old backup - left overs
// we may have to clean up old backup - leftovers
if (is_dir($dir.'/database_backup'))
{
self::remove_dir_content($dir.'/database_backup/');
@ -594,7 +594,10 @@ class Backup
$blobs = array();
foreach($this->schemas[$table]['fd'] as $col => $data)
{
if ($data['type'] == 'blob') $blobs[] = $col;
if (in_array($data['type'], ['blob', 'binary']))
{
$blobs[] = $col;
}
}
// check if we have an old PostgreSQL backup using 't'/'f' for bool values
// --> convert them to MySQL and our new PostgreSQL format of 1/0
@ -866,6 +869,7 @@ class Backup
case 'timestamp':
break;
case 'blob':
case 'binary':
$data = base64_encode($data);
break;
case 'bool': // we use MySQL 0, 1 in csv, not PostgreSQL 't', 'f'
@ -890,6 +894,7 @@ class Backup
* @param boolean $lock_table =null true: allways, false: never, null: if no primary key
* default of null limits memory usage if there is no primary key, by locking table and use ROW_CHUCK
* @param bool $skip_files_backup =false true: do not backup files, even if config / $this->backup_files is set (used by upgrade)
* @return string|true true: on success, string with error-message on failure
* @todo use https://github.com/maennchen/ZipStream-PHP to not assemble all files in memmory
*/
function backup($f, $lock_table=null, bool $skip_files_backup=false)
@ -1182,7 +1187,7 @@ class Backup
* @param string $file filename
* @param bool|string $restore true: 'Restore', false: 'Backup', string with custom label
* @param bool $start true: start of operation, false: end, null: neither
* @return void
* @param ?string $error_msg optional error-msg to log
*/
public function log(string $file, $restore, bool $start=null, string $error_msg=null)
{

View File

@ -130,6 +130,7 @@ class Schema
$this->max_varchar_length = 8000;
break;
case 'mysql':
case 'mysqli':
// since MySQL 5.0 65535, but with utf8 and row-size-limit of 64k:
// it's effective 65535/3 - size of other columns, so we use 20000 (mysql silently convert to text anyway)
if ((float)$this->m_odb->ServerInfo['version'] >= 5.0)
@ -1251,7 +1252,7 @@ class Schema
* The definition might not be as accurate, depending on the DB!
*
* @param string $sTableName table-name
* @return array|boolean table-defition, like $phpgw_baseline[$sTableName] after including tables_current, or false on error
* @return array|boolean table-definition, like $phpgw_baseline[$sTableName] after including tables_current, or false on error
*/
function GetTableDefinition($sTableName)
{
@ -1307,9 +1308,17 @@ class Schema
}
break;
case 'C': case 'C2':
// (var)binary column --> binary
if ($column->binary || $column->type === 'varbinary')
{
$definition['fd'][$name]['type'] = 'binary';
}
// ascii columns are reported as varchar
$definition['fd'][$name]['type'] = $this->m_odb->get_column_attribute($name, $sTableName, true, 'type') === 'ascii' ?
'ascii' : 'varchar';
else
{
$definition['fd'][$name]['type'] = $this->m_odb->get_column_attribute($name, $sTableName, true, 'type') === 'ascii' ?
'ascii' : 'varchar';
}
$definition['fd'][$name]['precision'] = $column->max_length;
break;
case 'B':

View File

@ -18,147 +18,8 @@ namespace EGroupware\Api\Vfs;
*
* @link http://www.php.net/manual/en/function.stream-wrapper-register.php
*/
interface StreamWrapperIface
interface StreamWrapperIface extends StreamWrapperIfaceNoDir
{
/**
* optional context param when opening the stream, null if no context passed
*
* @var mixed
*/
//var $context;
/**
* This method is called immediately after your stream object is created.
*
* @param string $path URL that was passed to fopen() and that this object is expected to retrieve
* @param string $mode mode used to open the file, as detailed for fopen()
* @param int $options additional flags set by the streams API (or'ed together):
* - STREAM_USE_PATH If path is relative, search for the resource using the include_path.
* - STREAM_REPORT_ERRORS If this flag is set, you are responsible for raising errors using trigger_error() during opening of the stream.
* If this flag is not set, you should not raise any errors.
* @param string $opened_path full path of the file/resource, if the open was successfull and STREAM_USE_PATH was set
* @return boolean true if the ressource was opened successful, otherwise false
*/
function stream_open ( $path, $mode, $options, &$opened_path );
/**
* This method is called when the stream is closed, using fclose().
*
* You must release any resources that were locked or allocated by the stream.
*/
function stream_close ( );
/**
* This method is called in response to fread() and fgets() calls on the stream.
*
* You must return up-to count bytes of data from the current read/write position as a string.
* If there are less than count bytes available, return as many as are available.
* If no more data is available, return either FALSE or an empty string.
* You must also update the read/write position of the stream by the number of bytes that were successfully read.
*
* @param int $count
* @return string/false up to count bytes read or false on EOF
*/
function stream_read ( $count );
/**
* This method is called in response to fwrite() calls on the stream.
*
* You should store data into the underlying storage used by your stream.
* If there is not enough room, try to store as many bytes as possible.
* You should return the number of bytes that were successfully stored in the stream, or 0 if none could be stored.
* You must also update the read/write position of the stream by the number of bytes that were successfully written.
*
* @param string $data
* @return integer
*/
function stream_write ( $data );
/**
* This method is called in response to feof() calls on the stream.
*
* Important: PHP 5.0 introduced a bug that wasn't fixed until 5.1: the return value has to be the oposite!
*
* if(version_compare(PHP_VERSION,'5.0','>=') && version_compare(PHP_VERSION,'5.1','<'))
* {
* $eof = !$eof;
* }
*
* @return boolean true if the read/write position is at the end of the stream and no more data availible, false otherwise
*/
function stream_eof ( );
/**
* This method is called in response to ftell() calls on the stream.
*
* @return integer current read/write position of the stream
*/
function stream_tell ( );
/**
* This method is called in response to fseek() calls on the stream.
*
* You should update the read/write position of the stream according to offset and whence.
* See fseek() for more information about these parameters.
*
* @param integer $offset
* @param integer $whence SEEK_SET - Set position equal to offset bytes
* SEEK_CUR - Set position to current location plus offset.
* SEEK_END - Set position to end-of-file plus offset. (To move to a position before the end-of-file, you need to pass a negative value in offset.)
* @return boolean TRUE if the position was updated, FALSE otherwise.
*/
function stream_seek ( $offset, $whence );
/**
* This method is called in response to fflush() calls on the stream.
*
* If you have cached data in your stream but not yet stored it into the underlying storage, you should do so now.
*
* @return boolean TRUE if the cached data was successfully stored (or if there was no data to store), or FALSE if the data could not be stored.
*/
function stream_flush ( );
/**
* This method is called in response to fstat() calls on the stream.
*
* If you plan to use your wrapper in a require_once you need to define stream_stat().
* If you plan to allow any other tests like is_file()/is_dir(), you have to define url_stat().
* stream_stat() must define the size of the file, or it will never be included.
* url_stat() must define mode, or is_file()/is_dir()/is_executable(), and any of those functions affected by clearstatcache() simply won't work.
* It's not documented, but directories must be a mode like 040777 (octal), and files a mode like 0100666.
* If you wish the file to be executable, use 7s instead of 6s.
* The last 3 digits are exactly the same thing as what you pass to chmod.
* 040000 defines a directory, and 0100000 defines a file.
*
* @return array containing the same values as appropriate for the stream.
*/
function stream_stat ( );
/**
* This method is called in response to unlink() calls on URL paths associated with the wrapper.
*
* It should attempt to delete the item specified by path.
* In order for the appropriate error message to be returned, do not define this method if your wrapper does not support unlinking!
*
* @param string $path
* @return boolean TRUE on success or FALSE on failure
*/
function unlink ( $path );
/**
* This method is called in response to rename() calls on URL paths associated with the wrapper.
*
* It should attempt to rename the item specified by path_from to the specification given by path_to.
* In order for the appropriate error message to be returned, do not define this method if your wrapper does not support renaming.
*
* The regular filesystem stream-wrapper returns an error, if $url_from and $url_to are not either both files or both dirs!
*
* @param string $path_from
* @param string $path_to
* @return boolean TRUE on success or FALSE on failure
*/
function rename ( $path_from, $path_to );
/**
* This method is called in response to mkdir() calls on URL paths associated with the wrapper.
*
@ -183,68 +44,4 @@ interface StreamWrapperIface
* @return boolean TRUE on success or FALSE on failure.
*/
function rmdir ( $path, $options );
/**
* This method is called immediately when your stream object is created for examining directory contents with opendir().
*
* @param string $path URL that was passed to opendir() and that this object is expected to explore.
* @return boolean
*/
function dir_opendir ( $path, $options );
/**
* This method is called in response to stat() calls on the URL paths associated with the wrapper.
*
* It should return as many elements in common with the system function as possible.
* Unknown or unavailable values should be set to a rational value (usually 0).
*
* If you plan to use your wrapper in a require_once you need to define stream_stat().
* If you plan to allow any other tests like is_file()/is_dir(), you have to define url_stat().
* stream_stat() must define the size of the file, or it will never be included.
* url_stat() must define mode, or is_file()/is_dir()/is_executable(), and any of those functions affected by clearstatcache() simply won't work.
* It's not documented, but directories must be a mode like 040777 (octal), and files a mode like 0100666.
* If you wish the file to be executable, use 7s instead of 6s.
* The last 3 digits are exactly the same thing as what you pass to chmod.
* 040000 defines a directory, and 0100000 defines a file.
*
* @param string $path
* @param int $flags holds additional flags set by the streams API. It can hold one or more of the following values OR'd together:
* - STREAM_URL_STAT_LINK For resources with the ability to link to other resource (such as an HTTP Location: forward,
* or a filesystem symlink). This flag specified that only information about the link itself should be returned,
* not the resource pointed to by the link.
* This flag is set in response to calls to lstat(), is_link(), or filetype().
* - STREAM_URL_STAT_QUIET If this flag is set, your wrapper should not raise any errors. If this flag is not set,
* you are responsible for reporting errors using the trigger_error() function during stating of the path.
* stat triggers it's own warning anyway, so it makes no sense to trigger one by our stream-wrapper!
* @return array
*/
function url_stat ( $path, $flags );
/**
* This method is called in response to readdir().
*
* It should return a string representing the next filename in the location opened by dir_opendir().
*
* @return string
*/
function dir_readdir ( );
/**
* This method is called in response to rewinddir().
*
* It should reset the output generated by dir_readdir(). i.e.:
* The next call to dir_readdir() should return the first entry in the location returned by dir_opendir().
*
* @return boolean
*/
function dir_rewinddir ( );
/**
* This method is called in response to closedir().
*
* You should release any resources which were locked or allocated during the opening and use of the directory stream.
*
* @return boolean
*/
function dir_closedir ( );
}
}

View File

@ -0,0 +1,225 @@
<?php
/**
* EGroupware API: VFS - stream wrapper interface
*
* @link http://www.egroupware.org
* @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
* @package api
* @subpackage vfs
* @version $Id$
*/
namespace EGroupware\Api\Vfs;
/**
* VFS - stream wrapper interface without mkdir and rmdir
*
* The interface is according to the docu on php.net
*
* @link http://www.php.net/manual/en/function.stream-wrapper-register.php
*/
interface StreamWrapperIfaceNoDir
{
/**
* optional context param when opening the stream, null if no context passed
*
* @var mixed
*/
//var $context;
/**
* This method is called immediately after your stream object is created.
*
* @param string $path URL that was passed to fopen() and that this object is expected to retrieve
* @param string $mode mode used to open the file, as detailed for fopen()
* @param int $options additional flags set by the streams API (or'ed together):
* - STREAM_USE_PATH If path is relative, search for the resource using the include_path.
* - STREAM_REPORT_ERRORS If this flag is set, you are responsible for raising errors using trigger_error() during opening of the stream.
* If this flag is not set, you should not raise any errors.
* @param string $opened_path full path of the file/resource, if the open was successfull and STREAM_USE_PATH was set
* @return boolean true if the ressource was opened successful, otherwise false
*/
function stream_open ( $path, $mode, $options, &$opened_path );
/**
* This method is called when the stream is closed, using fclose().
*
* You must release any resources that were locked or allocated by the stream.
*/
function stream_close ( );
/**
* This method is called in response to fread() and fgets() calls on the stream.
*
* You must return up-to count bytes of data from the current read/write position as a string.
* If there are less than count bytes available, return as many as are available.
* If no more data is available, return either FALSE or an empty string.
* You must also update the read/write position of the stream by the number of bytes that were successfully read.
*
* @param int $count
* @return string/false up to count bytes read or false on EOF
*/
function stream_read ( $count );
/**
* This method is called in response to fwrite() calls on the stream.
*
* You should store data into the underlying storage used by your stream.
* If there is not enough room, try to store as many bytes as possible.
* You should return the number of bytes that were successfully stored in the stream, or 0 if none could be stored.
* You must also update the read/write position of the stream by the number of bytes that were successfully written.
*
* @param string $data
* @return integer
*/
function stream_write ( $data );
/**
* This method is called in response to feof() calls on the stream.
*
* Important: PHP 5.0 introduced a bug that wasn't fixed until 5.1: the return value has to be the oposite!
*
* if(version_compare(PHP_VERSION,'5.0','>=') && version_compare(PHP_VERSION,'5.1','<'))
* {
* $eof = !$eof;
* }
*
* @return boolean true if the read/write position is at the end of the stream and no more data availible, false otherwise
*/
function stream_eof ( );
/**
* This method is called in response to ftell() calls on the stream.
*
* @return integer current read/write position of the stream
*/
function stream_tell ( );
/**
* This method is called in response to fseek() calls on the stream.
*
* You should update the read/write position of the stream according to offset and whence.
* See fseek() for more information about these parameters.
*
* @param integer $offset
* @param integer $whence SEEK_SET - Set position equal to offset bytes
* SEEK_CUR - Set position to current location plus offset.
* SEEK_END - Set position to end-of-file plus offset. (To move to a position before the end-of-file, you need to pass a negative value in offset.)
* @return boolean TRUE if the position was updated, FALSE otherwise.
*/
function stream_seek ( $offset, $whence );
/**
* This method is called in response to fflush() calls on the stream.
*
* If you have cached data in your stream but not yet stored it into the underlying storage, you should do so now.
*
* @return boolean TRUE if the cached data was successfully stored (or if there was no data to store), or FALSE if the data could not be stored.
*/
function stream_flush ( );
/**
* This method is called in response to fstat() calls on the stream.
*
* If you plan to use your wrapper in a require_once you need to define stream_stat().
* If you plan to allow any other tests like is_file()/is_dir(), you have to define url_stat().
* stream_stat() must define the size of the file, or it will never be included.
* url_stat() must define mode, or is_file()/is_dir()/is_executable(), and any of those functions affected by clearstatcache() simply won't work.
* It's not documented, but directories must be a mode like 040777 (octal), and files a mode like 0100666.
* If you wish the file to be executable, use 7s instead of 6s.
* The last 3 digits are exactly the same thing as what you pass to chmod.
* 040000 defines a directory, and 0100000 defines a file.
*
* @return array containing the same values as appropriate for the stream.
*/
function stream_stat ( );
/**
* This method is called in response to unlink() calls on URL paths associated with the wrapper.
*
* It should attempt to delete the item specified by path.
* In order for the appropriate error message to be returned, do not define this method if your wrapper does not support unlinking!
*
* @param string $path
* @return boolean TRUE on success or FALSE on failure
*/
function unlink ( $path );
/**
* This method is called in response to rename() calls on URL paths associated with the wrapper.
*
* It should attempt to rename the item specified by path_from to the specification given by path_to.
* In order for the appropriate error message to be returned, do not define this method if your wrapper does not support renaming.
*
* The regular filesystem stream-wrapper returns an error, if $url_from and $url_to are not either both files or both dirs!
*
* @param string $path_from
* @param string $path_to
* @return boolean TRUE on success or FALSE on failure
*/
function rename ( $path_from, $path_to );
/**
* This method is called immediately when your stream object is created for examining directory contents with opendir().
*
* @param string $path URL that was passed to opendir() and that this object is expected to explore.
* @return boolean
*/
function dir_opendir ( $path, $options );
/**
* This method is called in response to stat() calls on the URL paths associated with the wrapper.
*
* It should return as many elements in common with the system function as possible.
* Unknown or unavailable values should be set to a rational value (usually 0).
*
* If you plan to use your wrapper in a require_once you need to define stream_stat().
* If you plan to allow any other tests like is_file()/is_dir(), you have to define url_stat().
* stream_stat() must define the size of the file, or it will never be included.
* url_stat() must define mode, or is_file()/is_dir()/is_executable(), and any of those functions affected by clearstatcache() simply won't work.
* It's not documented, but directories must be a mode like 040777 (octal), and files a mode like 0100666.
* If you wish the file to be executable, use 7s instead of 6s.
* The last 3 digits are exactly the same thing as what you pass to chmod.
* 040000 defines a directory, and 0100000 defines a file.
*
* @param string $path
* @param int $flags holds additional flags set by the streams API. It can hold one or more of the following values OR'd together:
* - STREAM_URL_STAT_LINK For resources with the ability to link to other resource (such as an HTTP Location: forward,
* or a filesystem symlink). This flag specified that only information about the link itself should be returned,
* not the resource pointed to by the link.
* This flag is set in response to calls to lstat(), is_link(), or filetype().
* - STREAM_URL_STAT_QUIET If this flag is set, your wrapper should not raise any errors. If this flag is not set,
* you are responsible for reporting errors using the trigger_error() function during stating of the path.
* stat triggers it's own warning anyway, so it makes no sense to trigger one by our stream-wrapper!
* @return array
*/
function url_stat ( $path, $flags );
/**
* This method is called in response to readdir().
*
* It should return a string representing the next filename in the location opened by dir_opendir().
*
* @return string
*/
function dir_readdir ( );
/**
* This method is called in response to rewinddir().
*
* It should reset the output generated by dir_readdir(). i.e.:
* The next call to dir_readdir() should return the first entry in the location returned by dir_opendir().
*
* @return boolean
*/
function dir_rewinddir ( );
/**
* This method is called in response to closedir().
*
* You should release any resources which were locked or allocated during the opening and use of the directory stream.
*
* @return boolean
*/
function dir_closedir ( );
}

View File

@ -32,7 +32,7 @@ trait UserContextTrait
public $context;
/**
* Contructor to set context/user incl. from user in url or passed in context
* Constructor to set context/user incl. from user in url or passed in context
*
* @param resource|string|null $url_or_context url with user or context to set
*/

View File

@ -79,7 +79,7 @@
"bower-asset/jquery": "^1.12.4",
"defuse/php-encryption": "^2.4",
"egroupware/activesync": "self.version",
"egroupware/adodb-php": "^5.20.8",
"egroupware/adodb-php": "^5.20.9",
"egroupware/bookmarks": "self.version",
"egroupware/collabora": "self.version",
"egroupware/compress": "^2.3.0",
@ -159,4 +159,4 @@
},
"minimum-stability": "dev",
"prefer-stable": true
}
}

18
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "a1ebad77744e2ba1fadcf06b93dde612",
"content-hash": "0f2d8bfa2ed3bde041af7da260016181",
"packages": [
{
"name": "adldap2/adldap2",
@ -834,16 +834,16 @@
},
{
"name": "egroupware/adodb-php",
"version": "5.20.8",
"version": "5.20.9",
"source": {
"type": "git",
"url": "https://github.com/EGroupware/ADOdb.git",
"reference": "7c926a1e8435a172a0947bbd54dd33c6388ee3ba"
"reference": "695e4d98fd18869056aa06be14eb9ed248580d1b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/EGroupware/ADOdb/zipball/7c926a1e8435a172a0947bbd54dd33c6388ee3ba",
"reference": "7c926a1e8435a172a0947bbd54dd33c6388ee3ba",
"url": "https://api.github.com/repos/EGroupware/ADOdb/zipball/695e4d98fd18869056aa06be14eb9ed248580d1b",
"reference": "695e4d98fd18869056aa06be14eb9ed248580d1b",
"shasum": ""
},
"require": {
@ -884,7 +884,11 @@
"library",
"php"
],
"time": "2023-01-30T18:10:47+00:00"
"support": {
"issues": "https://github.com/ADOdb/ADOdb/issues",
"source": "https://github.com/ADOdb/ADOdb"
},
"time": "2023-10-30T11:00:40+00:00"
},
{
"name": "egroupware/bookmarks",
@ -11971,4 +11975,4 @@
"php": "7.4"
},
"plugin-api-version": "2.2.0"
}
}