* All apps/REST API: fix custom-fields of type "date-time" to be stored timezone aware, if no format is specified

So far date-time values were stored in user-time, now they are stored in UTC with a "Z" suffix" to be able to still read old user-time values unchanged.

api/src/CalDAV/JsBase.php

@@ -149,9 +149,10 @@ class JsBase
 	 *
 	 * @param array $contact
 	 * @param ?string $app default self::APP
+	 * @param ?string $timezone optional timezone-name to use für date-time types, default UTC
 	 * @return array
 	 */
-	protected static function customfields(array $contact, ?string $app=null)
+	protected static function customfields(array $contact, ?string $app=null, ?string $timezone=null)
 	{
 		$fields = [];
 		foreach(Api\Storage\Customfields::get($app ?? static::APP) as $name => $data)
@@ -162,7 +163,7 @@ class JsBase
 				switch($data['type'])
 				{
 					case 'date-time':
-						$value = Api\DateTime::to($value, Api\DateTime::RFC3339);
+						$value = empty($timezone) ? self::UTCDateTime($value) : self::DateTime($value, $timezone);
 						break;
 					case 'float':
 						$value = (double)$value;
@@ -193,9 +194,10 @@ class JsBase
 	 *
 	 * @param array $cfs name => object with attribute data and optional type, label, values
 	 * @param ?string $app default self::APP
+	 * @param ?string $timeZone timezone-name given in JSON data
 	 * @return array
 	 */
-	protected static function parseCustomfields(array $cfs, ?string $app=null)
+	protected static function parseCustomfields(array $cfs, ?string $app=null, ?string $timeZone=null)
 	{
 		$contact = [];
 		$definitions = Api\Storage\Customfields::get($app ?? static::APP);
@@ -216,7 +218,7 @@ class JsBase
 				switch($definition['type'])
 				{
 					case 'date-time':
-						$data['value'] = Api\DateTime::to($data['value'], 'object');
+						$data['value'] = self::parseDateTime($data['value'], $timeZone);
 						break;
 					case 'float':
 						$data['value'] = (double)$data['value'];

api/src/CalDAV/JsCalendar.php

@@ -68,7 +68,7 @@ class JsCalendar extends JsBase
 			'priority' => self::Priority($event['priority']),
 			'categories' => self::categories($event['category']),
 			'privacy' => $event['public'] ? 'public' : 'private',
-			'egroupware.org:customfields' => self::customfields($event),
+			'egroupware.org:customfields' => self::customfields($event, null, $event['tzid']),
 		] + self::Locations($event);
 
 		if (!empty($event['recur_type']) || $exceptions)
@@ -113,12 +113,12 @@ class JsCalendar extends JsBase
 				}))
 			{
 				// apply patch on JsEvent
-				$data = self::patch($data, $old ? self::getJsCalendar($old, false) : [], !$old || !$strict);
+				$data = self::patch($data, $old ? self::JsEvent($old, false) : [], !$old || !$strict);
 			}
 
 			if (!isset($data['uid'])) $data['uid'] = null;  // to fail below, if it does not exist
 
-			$event = [];
+			$event = $old ? ['id' => $old['id']] : [];
 			foreach ($data as $name => $value)
 			{
 				switch ($name)
@@ -181,7 +181,7 @@ class JsCalendar extends JsBase
 						break;
 
 					case 'egroupware.org:customfields':
-						$event = array_merge($event, self::parseCustomfields($value, $strict));
+						$event = array_merge($event, self::parseCustomfields($value, 'calendar', $data['timeZone']));
 						break;
 
 					case 'prodId':
@@ -256,7 +256,7 @@ class JsCalendar extends JsBase
 				'egroupware.org:completed' => $entry['info_datecomplete'] ?
 					self::DateTime($entry['info_datecompleted'], Api\DateTime::$user_timezone->getName()) : null,
 			] + self::Locations(['location' => $entry['info_location'] ?? null]) + self::relatedToParent($entry['info_id_parent']) + [
-				'egroupware.org:customfields' => self::customfields($entry, 'infolog'),
+				'egroupware.org:customfields' => self::customfields($entry, 'infolog', Api\DateTime::$user_timezone->getName()),
 			];
 
 		if (!empty($entry['##RRULE']))
@@ -301,12 +301,12 @@ class JsCalendar extends JsBase
 				}))
 			{
 				// apply patch on JsEvent
-				$data = self::patch($data, $old ? self::getJsTask($old, false) : [], !$old || !$strict);
+				$data = self::patch($data, $old ? self::JsTask($old, false) : [], !$old || !$strict);
 			}
 
 			if (!isset($data['uid'])) $data['uid'] = null;  // to fail below, if it does not exist
 
-			$event = [];
+			$event = $old ? ['info_id' => $old['info_id']] : [];
 			foreach ($data as $name => $value)
 			{
 				switch ($name)
@@ -368,7 +368,7 @@ class JsCalendar extends JsBase
 						break;
 
 					case 'egroupware.org:customfields':
-						$event = array_merge($event, self::parseCustomfields($value, 'infolog'));
+						$event = array_merge($event, self::parseCustomfields($value, 'infolog', $data['timeZone']));
 						break;
 
 					case 'egroupware.org:type':

api/src/Contacts/Storage.php

@@ -273,8 +273,9 @@ class Storage
 				$this->contact_repository = 'sql-ldap';
 			}
 			$this->somain = new Sql($db);
+			$this->somain->add_cf_timestamps();
 
-			// remove some columns, absolutly not necessary to search in sql
+			// remove some columns, absolutely not necessary to search in sql
 			$this->columns_to_search = array_diff(array_values($this->somain->db_cols),$this->sql_cols_not_to_search);
 		}
 		$this->grants = $this->get_grants($this->user,$contact_app);

api/src/DateTime.php

@@ -33,7 +33,7 @@ use DateInterval;
  * - Api\DateTime::user2server($time,$type=null)
  * (Replacing in 1.6 and previous used adding of tz_offset, which is only correct for current time)
  *
- * An other static method allows to format any time in several ways: Api\DateTime::to($time,$type) (exceed date($type,$time)).
+ * Another static method allows to format any time in several ways: Api\DateTime::to($time,$type) (exceed date($type,$time)).
  *
  * The constructor of Api\DateTime understand - in addition to DateTime - integer timestamps, array with values for
  * keys: ('year', 'month', 'day') or 'full' plus 'hour', 'minute' and optional 'second' or a DateTime object as parameter.
@@ -122,7 +122,7 @@ class DateTime extends \DateTime
 						break;
 					}
 					catch(Exception $e) {
-						// if string is nummeric, ignore the exception and treat string as timestamp
+						// if string is numeric, ignore the exception and treat string as timestamp
 						if (!is_numeric($time)) throw $e;
 					}
 				}

api/src/Storage.php

@@ -172,6 +172,27 @@ class Storage extends Storage\Base
 		$this->customfields = Storage\Customfields::get($app, false, null, $db);
 	}
 
+	/**
+	 * Add all CFs of type date-time to $this->timestamps to get automatically converted to and from usertime
+	 */
+	function convert_all_timestamps()
+	{
+		parent::convert_all_timestamps();
+
+		$this->add_cf_timestamps();
+	}
+
+	function add_cf_timestamps()
+	{
+		foreach($this->customfields ?? [] as $name => $cf)
+		{
+			if ($cf['type'] === 'date-time' && !in_array($field=$this->get_cf_field($name), $this->timestamps))
+			{
+				$this->timestamps[] = $field;
+			}
+		}
+	}
+
 	/**
 	 * Read all customfields of the given id's
 	 *
@@ -203,6 +224,12 @@ class Storage extends Storage\Base
 			{
 				$entry[$field][] = $row[$this->extra_value];
 			}
+			// old date-time CFs are stored in user-time, new ones in UTC with "Z" suffix, we always return them now as DateTime objects
+			elseif (($this->customfields[$row[$this->extra_key]]['type']??null) === 'date-time' &&
+				empty($this->customfields[$row[$this->extra_key]]['values']['format']))  // but only if they have no format specified
+			{
+				$entry[$field] = new DateTime($row[$this->extra_value], DateTime::$user_timezone);
+			}
 			else
 			{
 				$entry[$field] = $row[$this->extra_value];
@@ -224,7 +251,7 @@ class Storage extends Storage\Base
 
 		Customfields::handle_files($this->app, $id, $data, $this->customfields);
 
-		foreach (array_keys((array)$this->customfields) as $name)
+		foreach ((array)$this->customfields as $name => $cf)
 		{
 			if (!isset($data[$field = $this->get_cf_field($name)])) continue;
 
@@ -240,10 +267,17 @@ class Storage extends Storage\Base
 				$this->db->delete($this->extra_table,$where,__LINE__,__FILE__,$this->app);
 				if (empty($data[$field])) continue;	// nothing else to do for empty values
 			}
-			foreach($is_multiple && !is_array($data[$field]) ? explode(',',$data[$field]) :
+			foreach($is_multiple && is_string($data[$field]) ? explode(',',$data[$field]) :
 				// regular custom fields (!$is_multiple) eg. addressbook store multiple values comma-separated
-				(array)(!$is_multiple && is_array($data[$field]) ? implode(',', $data[$field]) : $data[$field]) as $value)
+				[!$is_multiple && is_array($data[$field]) ? implode(',', $data[$field]) : $data[$field]] as $value)
 			{
+				// store type date-time in UTC with "Z" suffix, to be able to distinguish them from old date-time stored in user-time!
+				if ($cf['type'] === 'date-time' && empty($cf['values']['format']))  // but only if they have no format specified
+				{
+					$time = new DateTime($value, DateTime::$server_timezone);
+					$time->setTimezone(new \DateTimeZone('UTC'));
+					$value = $time->format('Y-m-d H:i:s').'Z';
+				}
 				if (!$this->db->insert($this->extra_table,array($this->extra_value => $value)+$extra_cols,$where,__LINE__,__FILE__,$this->app))
 				{
 					return $this->db->Errno;
@@ -320,7 +354,17 @@ class Storage extends Storage\Base
 
 		if ($ret == 0 && $this->customfields)
 		{
-			$this->save_customfields($this->data);
+			// if we have date-time custom-fields, we have to convert them from user-time
+			if ($this->timestamps && $this->is_cf(end($this->timestamps)))
+			{
+				$this->data2db();   // save has already reverted timezone of timestamps again back to user-time
+				$this->save_customfields($this->data);
+				$this->db2data();
+			}
+			else
+			{
+				$this->save_customfields($this->data);
+			}
 		}
 		return $ret;
 	}

api/src/Storage/Base.php

@@ -121,14 +121,14 @@ class Base
 	 * Possible values:
 	 * - 'ts'|'integer' convert every timestamp to an integer unix timestamp
 	 * - 'string' convert every timestamp to a 'Y-m-d H:i:s' string
-	 * - 'object' convert every timestamp to a Api\DateTime object
+	 * - 'object' convert every timestamp to an Api\DateTime object
 	 *
 	 * @var string
 	 */
 	public $timestamp_type;
 	/**
-	 * Offset in secconds between user and server-time,	it need to be add to a server-time to get the user-time
+	 * Offset in seconds between user and server-time,	it need to be add to a server-time to get the user-time
-	 * or substracted from a user-time to get the server-time
+	 * or subtracted from a user-time to get the server-time
 	 *
 	 * @var int
 	 * @deprecated use Api\DateTime methods instead, as the offset between user and server time is only valid for current time

calendar/inc/class.calendar_bo.inc.php

@@ -1018,11 +1018,11 @@ class calendar_bo
 				$timestamps = array('start','end','modified','created','recur_enddate','recurrence','recur_date','deleted');
 			}
 			// we convert here from the server-time timestamps to user-time and (optional) to a different date-format!
-			foreach ($timestamps as $ts)
+			foreach (array_merge($timestamps, $this->getCfTtimestamps()) as $ts)
 			{
 				if (!empty($event[$ts]))
 				{
-					$event[$ts] = $this->date2usertime((int)$event[$ts],$date_format);
+					$event[$ts] = $this->date2usertime($event[$ts], $date_format);
 				}
 			}
 			// same with the recur exceptions and rdates
@@ -1054,6 +1054,22 @@ class calendar_bo
 		}
 	}
 
+	public function getCfTtimestamps()
+	{
+		static $cf_timestamps=null;
+		if (!isset($cf_timestamps))
+		{
+			$cf_timestamps = array_map(static function($name)
+			{
+				return '#' . $name;
+			}, array_keys(array_filter($this->customfields?:[], static function ($cf)
+			{
+				return $cf['type'] === 'date-time';
+			})));
+		}
+		return $cf_timestamps;
+	}
+
 	/**
 	 * convert a date from server to user-time
 	 *

calendar/inc/class.calendar_boupdate.inc.php

@@ -1601,7 +1601,7 @@ class calendar_boupdate extends calendar_bo
 			$timestamps = array('start','end','modified','created','recur_enddate','recurrence');
 		}
 		// we run all dates through date2ts, to adjust to server-time and the possible date-formats
-		foreach($timestamps as $ts)
+		foreach(array_merge($timestamps, $this->getCfTtimestamps()) as $ts)
 		{
 			// we convert here from user-time to timestamps in server-time!
 			if (isset($event[$ts])) $event[$ts] = $event[$ts] ? $this->date2ts($event[$ts],true) : 0;

calendar/inc/class.calendar_so.inc.php

@@ -133,6 +133,8 @@ class calendar_so
 	 */
 	protected static $tz_cache = array();
 
+	protected $customfields;
+
 	/**
 	 * Constructor of the socal class
 	 */
@@ -147,6 +149,7 @@ class calendar_so
 			$vname = $name.'_table';
 			$this->all_tables[] = $this->$vname = $this->cal_table.'_'.$name;
 		}
+		$this->customfields = Api\Storage\Customfields::get('calendar');
 	}
 
 	/**
@@ -512,7 +515,16 @@ class calendar_so
 		// custom fields
 		foreach($this->db->select($this->extra_table,'*',array('cal_id'=>$ids),__LINE__,__FILE__,false,'','calendar') as $row)
 		{
-			$events[$row['cal_id']]['#'.$row['cal_extra_name']] = $row['cal_extra_value'];
+			// old date-time CFs are stored in user-time, new ones in UTC with "Z" suffix, we always return them now as DateTime objects
+			if (($this->customfields[$row['cal_extra_name']]['type']??null) === 'date-time' &&
+				empty($this->customfields[$row['cal_extra_name']]['values']['format']))  // but only if they have no format specified)
+			{
+				$events[$row['cal_id']]['#'.$row['cal_extra_name']] = new Api\DateTime($row['cal_extra_value'], Api\DateTime::$user_timezone);
+			}
+			else
+			{
+				$events[$row['cal_id']]['#'.$row['cal_extra_name']] = $row['cal_extra_value'];
+			}
 		}
 
 		// alarms
@@ -1507,7 +1519,7 @@ ORDER BY cal_user_type, cal_usre_id
 
 		$event['cal_category'] = implode(',',$categories);
 
-		// make sure recurring events never reference to an other recurrent event
+		// make sure recurring events never reference to another recurrent event
 		if (!empty($event['recur_type'])) $event['cal_reference'] = 0;
 
 		if ($cal_id)
@@ -1611,7 +1623,7 @@ ORDER BY cal_user_type, cal_usre_id
 				}
 			}
 		}
-		else // write information about recuring event, if recur_type is present in the array
+		else // write information about recurring event, if recur_type is present in the array
 		{
 			// fetch information about the currently saved (old) event
 			$old_min = (int) $this->db->select($this->dates_table,'MIN(cal_start)',array('cal_id'=>$cal_id),__LINE__,__FILE__,false,'','calendar')->fetchColumn();
@@ -1763,6 +1775,14 @@ ORDER BY cal_user_type, cal_usre_id
 				}
 				if ($value)
 				{
+					// store type date-time in UTC with "Z" suffix, to be able to distinguish them from old date-time stored in user-time!
+					if (($this->customfields[substr($name, 1)]['type']??null) === 'date-time' &&
+						empty($this->customfields[substr($name, 1)]['values']['format']))  // but only if they have no format specified)
+					{
+						$time = new Api\DateTime($value, Api\DateTime::$server_timezone);
+						$time->setTimezone(new DateTimeZone('UTC'));
+						$value = $time->format('Y-m-d H:i:s').'Z';
+					}
 					$this->db->insert($this->extra_table,array(
 						'cal_extra_value'	=> is_array($value) ? implode(',',$value) : $value,
 					),array(

infolog/inc/class.infolog_bo.inc.php

@@ -141,7 +141,7 @@ class infolog_bo
 	var $tracking;
 	/**
 	 * Maximum number of line characters (-_+=~) allowed in a mail, to not stall the layout.
-	 * Longer lines / biger number of these chars are truncated to that max. number or chars.
+	 * Longer lines / bigger number of these chars are truncated to that max. number or chars.
 	 *
 	 * @var int
 	 */
@@ -270,6 +270,11 @@ class infolog_bo
 						$this->customfields[$name] = $field;
 						$save_config = true;
 					}
+					// add date-time CFs to timestamps to ensure TZ conversation
+					if ($field['type'] === 'date-time')
+					{
+						$this->timestamps[] = '#'.$field['name'];
+					}
 				}
 				if (!empty($save_config)) Api\Config::save_value('customfields',$this->customfields,'infolog');
 			}
@@ -554,9 +559,6 @@ class infolog_bo
 	 */
 	 function time2time(&$values, $fromTZId=false, $toTZId=null, $type='ts')
 	 {
-
-		if ($fromTZId === $toTZId) return;
-
 		$tz = Api\DateTime::$server_timezone;
 
 	 	if ($fromTZId)
@@ -634,9 +636,9 @@ class infolog_bo
 	 *
 	 * @param int|array $info_id integer id or array with id's or array with column=>value pairs of the entry to read
 	 * @param boolean $run_link_id2from = true should link_id2from run, default yes,
-	 *	need to be set to false if called from link-title to prevent an infinit recursion
+	 *	need to be set to false if called from link-title to prevent an infinite recursion
 	 * @param string $date_format = 'ts' date-formats: 'ts'=timestamp, 'server'=timestamp in server-time,
-	 * 	'array'=array or string with date-format
+	 * 	'array'=array or string with date-format, 'object' DateTime objects
 	 * @param boolean $ignore_acl = false if true, do NOT check access, default false
 	 *
 	 * @return array|boolean infolog entry, null if not found or false if no permission to read it

infolog/inc/class.infolog_so.inc.php

@@ -69,6 +69,10 @@ class infolog_so
 	 * @var int
 	 */
 	 var $tz_offset;
+	/**
+	 * @var array $customfields if defined
+	 */
+	 protected array $customfields;
 
 	/**
 	 * Constructor
@@ -84,6 +88,8 @@ class infolog_so
 		$this->user   = $GLOBALS['egw_info']['user']['account_id'];
 
 		$this->tz_offset = $GLOBALS['egw_info']['user']['preferences']['common']['tz_offset'];
+
+		$this->customfields = Api\Storage\Customfields::get('infolog');
 	}
 
 	/**
@@ -448,7 +454,16 @@ class infolog_so
 		$this->data['info_id_parent'] = (int)$this->data['info_id_parent'];
 		foreach($this->db->select($this->extra_table,'info_extra_name,info_extra_value',array('info_id'=>$this->data['info_id']),__LINE__,__FILE__) as $row)
 		{
-			$this->data['#'.$row['info_extra_name']] = $row['info_extra_value'];
+			// old date-time CFs are stored in user-time, new ones in UTC with "Z" suffix, we always return them now as DateTime objects
+			if (($this->customfields[$row['info_extra_name']]['type']??null) === 'date-time' &&
+				empty($this->customfields[$row['info_extra_name']]['values']['format']))  // but only if they have no format specified)
+			{
+				$this->data['#'.$row['info_extra_name']] = new Api\DateTime($row['info_extra_value'], Api\DateTime::$user_timezone);
+			}
+			else
+			{
+				$this->data['#'.$row['info_extra_name']] = $row['info_extra_value'];
+			}
 		}
 		//error_log(__METHOD__.'('.array2string($where).') returning '.array2string($this->data));
 		return $this->data;
@@ -663,6 +678,14 @@ class infolog_so
 
 			if ($val)
 			{
+				// store type date-time in UTC with "Z" suffix, to be able to distinguish them from old date-time stored in user-time!
+				if (($this->customfields[substr($key, 1)]['type']??null) === 'date-time' &&
+					empty($this->customfields[substr($key, 1)]['values']['format']))  // but only if they have no format specified)
+				{
+					$time = new Api\DateTime($val, Api\DateTime::$server_timezone);
+					$time->setTimezone(new DateTimeZone('UTC'));
+					$val = $time->format('Y-m-d H:i:s').'Z';
+				}
 				$this->db->insert($this->extra_table,array(
 						// store multivalued CalDAV properties as serialized array, everything else get comma-separated
 						'info_extra_value'	=> is_array($val) ? ($key[1] == '#' ? json_encode($val) : implode(',',$val)) : $val,
@@ -1054,7 +1077,16 @@ class infolog_so
 				}
 				foreach($this->db->select($this->extra_table,'*',$where,__LINE__,__FILE__) as $row)
 				{
-					$ids[$row['info_id']]['#'.$row['info_extra_name']] = $row['info_extra_value'];
+					// old date-time CFs are stored in user-time, new ones in UTC with "Z" suffix, we always return them now as DateTime objects
+					if (($this->customfields[$row['info_extra_name']]['type']??null) === 'date-time' &&
+						empty($this->customfields[$row['info_extra_name']]['values']['format']))  // but only if they have no format specified)
+					{
+						$ids[$row['info_id']]['#'.$row['info_extra_name']] = new Api\DateTime($row['info_extra_value'], Api\DateTime::$user_timezone);
+					}
+					else
+					{
+						$ids[$row['info_id']]['#'.$row['info_extra_name']] = $row['info_extra_value'];
+					}
 				}
 			}
 		}

timesheet/inc/class.timesheet_bo.inc.php

@@ -51,14 +51,6 @@ class timesheet_bo extends Api\Storage
 	 * @var int
 	 */
 	var $user;
-	/**
-	 * Timestaps that need to be adjusted to user-time on reading or saving
-	 *
-	 * @var array
-	 */
-	var $timestamps = array(
-		'ts_start','ts_created', 'ts_modified'
-	);
 	/**
 	 * Start of today in user-time
 	 *
@@ -170,6 +162,8 @@ class timesheet_bo extends Api\Storage
 	{
 		parent::__construct(TIMESHEET_APP,self::TABLE,self::EXTRA_TABLE,'','ts_extra_name','ts_extra_value','ts_id');
 
+		$this->convert_all_timestamps();
+
 		$this->config_data = Api\Config::read(TIMESHEET_APP);
 		$this->quantity_sum = $this->config_data['quantity_sum'] == 'true';
 
@@ -1069,7 +1063,7 @@ class timesheet_bo extends Api\Storage
 		{
 			$data =& $this->data;
 		}
-		// allways store ts_project to be able to search for it, even if no custom project is set
+		// always store ts_project to be able to search for it, even if no custom project is set
 		if (empty($data['ts_project']) && !is_null($data['ts_project']))
 		{
 			$data['ts_project'] = $data['pm_id'] ? Link::title('projectmanager', $data['pm_id']) : '';

timesheet/src/JsTimesheet.php

@@ -123,7 +123,7 @@ class JsTimesheet extends Api\CalDAV\JsBase
 						break;
 
 					case 'start':
-						$timesheet['ts_start'] = Api\DateTime::server2user($value, 'ts');
+						$timesheet['ts_start'] = self::parseDateTime($value);
 						break;
 
 					case 'duration':