* CalDAV/CardDAV/REST API: fix not working limited result

(since using generator instead of iterator)

addressbook/inc/class.addressbook_groupdav.inc.php

@@ -170,31 +170,22 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 			}
 		}
 		// rfc 6578 sync-collection report: filter for sync-token is already set in _report_filters
-		if ($options['root']['name'] == 'sync-collection')
+		if ($options['root']['name'] === 'sync-collection')
 		{
 			// callback to query sync-token, after propfind_callbacks / iterator is run and
 			// stored max. modification-time in $this->sync_collection_token
 			$files['sync-token'] = array($this, 'get_sync_collection_token');
 			$files['sync-token-params'] = array($path, $user);
 
-			$this->sync_collection_token = null;
+			$this->sync_collection_token = $this->more_results = null;
 
 			$filter['order'] = 'contact_modified ASC';	// return oldest modifications first
 			$filter['sync-collection'] = true;
 		}
 
-		if (isset($nresults))
+		if (isset($nresults) && $options['root']['name'] === 'sync-collection')
 		{
 			$files['files'] = $this->propfind_generator($path, $filter, $files['files'], (int)$nresults);
-
-			// hack to support limit with sync-collection report: contacts are returned in modified ASC order (oldest first)
-			// if limit is smaller than full result, return modified-1 as sync-token, so client requests next chunk incl. modified
-			// (which might contain further entries with identical modification time)
-			if ($options['root']['name'] == 'sync-collection' && $this->bo->total > $nresults)
-			{
-				--$this->sync_collection_token;
-				$files['sync-token-params'][] = true;	// tell get_sync_collection_token that we have more entries
-			}
 		}
 		else
 		{
@@ -232,6 +223,8 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 		{
 			if (++$yielded && isset($nresults) && $yielded > $nresults)
 			{
+				$this->sync_collection_token = $resource['modified'];
+				$this->more_results = true;
 				return;
 			}
 			yield $resource;
@@ -307,13 +300,15 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 				{
 					unset($this->requested_multiget_ids[$k]);
 				}
+				if (++$yielded && isset($nresults) && $yielded > $nresults)
+				{
+					$this->sync_collection_token = $contact['modified'];
+					$this->more_results = true;
+					return;
+				}
 				// sync-collection report: deleted entry need to be reported without properties
 				if ($contact['tid'] == Api\Contacts::DELETED_TYPE)
 				{
-					if (++$yielded && isset($nresults) && $yielded > $nresults)
-					{
-						return;
-					}
 					yield ['path' => $path.urldecode($this->get_path($contact))];
 					continue;
 				}
@@ -329,10 +324,6 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 					$props['getcontentlength'] = bytes(is_array($content) ? json_encode($content) : $content);
 					$props['address-data'] = Api\CalDAV::mkprop(Api\CalDAV::CARDDAV, 'address-data', $content);
 				}
-				if (++$yielded && isset($nresults) && $yielded > $nresults)
-				{
-					return;
-				}
 				yield $this->add_resource($path, $contact, $props);
 			}
 			// sync-collection report --> return modified of last contact as sync-token
@@ -354,6 +345,8 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 			{
 				if (++$yielded && isset($nresults) && $yielded > $nresults)
 				{
+					$this->sync_collection_token = Api\DateTime::user2server($resource['modified'])-1;
+					$this->more_results = true;
 					return;
 				}
 				yield $resource;
@@ -408,6 +401,8 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 					}
 					if (++$yielded && isset($nresults) && $yielded > $nresults)
 					{
+						$this->sync_collection_token = $GLOBALS['egw']->db->from_timestamp($list['list_modified'])-1;
+						$this->more_results = true;
 						return;
 					}
 					yield $this->add_resource($path, $list, $props);
@@ -433,6 +428,8 @@ class addressbook_groupdav extends Api\CalDAV\Handler
 			{
 				if (++$yielded && isset($nresults) && $yielded > $nresults)
 				{
+					--$this->sync_collection_token;
+					$this->more_results = true;
 					return;
 				}
 				yield ['path' => $path.$id.self::$path_extension];

api/src/CalDAV/Handler.php

@@ -724,6 +724,10 @@ abstract class Handler
 	 * sync-token to be filled by propfind_callback and returned by get_sync_token method
 	 */
 	protected $sync_collection_token;
+	/**
+	 * @var bool true if result was limited by nresults parameter
+	 */
+	protected $more_results;
 
 	/**
 	 * Query sync-token from a just run sync-collection report
@@ -734,10 +738,10 @@ abstract class Handler
 	 * @param int $user parameter necessary to call getctag, if no $token specified
 	 * @return string
 	 */
-	public function get_sync_collection_token($path, $user=null, $more_results=null)
+	public function get_sync_collection_token($path, $user=null)
 	{
 		//error_log(__METHOD__."('$path', $user, more_results=$more_results) this->sync_collection_token=".$this->sync_collection_token);
-		if ($more_results)
+		if ($this->more_results)
 		{
 			if (Api\CalDAV::isJSON())
 			{

calendar/inc/class.calendar_groupdav.inc.php

@@ -238,14 +238,14 @@ class calendar_groupdav extends Api\CalDAV\Handler
 		}
 
 		// rfc 6578 sync-collection report: filter for sync-token is already set in _report_filters
-		if ($options['root']['name'] == 'sync-collection')
+		if ($options['root']['name'] === 'sync-collection')
 		{
 			// callback to query sync-token, after propfind_callbacks / iterator is run and
 			// stored max. modification-time in $this->sync_collection_token
 			$files['sync-token'] = array($this, 'get_sync_collection_token');
 			$files['sync-token-params'] = array($path, $user);
 
-			$this->sync_collection_token = null;
+			$this->sync_collection_token = $this->more_results = null;
 
 			$filter['order'] = 'cal_modified ASC';	// return oldest modifications first
 			$filter['sync-collection'] = true;
@@ -267,19 +267,10 @@ class calendar_groupdav extends Api\CalDAV\Handler
 			}
 		}
 
-		if (isset($nresults))
+		if (isset($nresults) && $options['root']['name'] === 'sync-collection')
 		{
 			unset($filter['no_total']);	// we need the total!
 			$files['files'] = $this->propfind_generator($path, $filter, $files['files'], (int)$nresults);
-
-			// hack to support limit with sync-collection report: events are returned in modified ASC order (oldest first)
-			// if limit is smaller than full result, return modified-1 as sync-token, so client requests next chunk incl. modified
-			// (which might contain further entries with identical modification time)
-			if ($options['root']['name'] == 'sync-collection' && $this->bo->total > $nresults)
-			{
-				--$this->sync_collection_token;
-				$files['sync-token-params'][] = true;	// tel get_sync_collection_token that we have more entries
-			}
 		}
 		else
 		{
@@ -366,6 +357,8 @@ class calendar_groupdav extends Api\CalDAV\Handler
 		{
 			if (++$yielded && isset($nresults) && $yielded > $nresults)
 			{
+				$this->sync_collection_token = Api\DateTime::user2server($resource['modified'], 'ts')-1;
+				$this->more_results = true;
 				return;
 			}
 			yield $resource;
@@ -385,15 +378,17 @@ class calendar_groupdav extends Api\CalDAV\Handler
 			{
 				$no_active_participants = !$this->hasActiveParticipants($event, $filter['users'], $exceptions);
 
+				if (++$yielded && isset($nresults) && $yielded > $nresults)
+				{
+					$this->sync_collection_token = Api\DateTime::user2server($event['modified'], 'ts')-1;
+					$this->more_results = true;
+					return;
+				}
 				// sync-collection report: deleted entries need to be reported without properties, same for rejected or deleted invitations
 				if ($sync_collection && ($event['deleted'] && !$event['cal_reference'] ||
 					//in_array($event['participants'][$filter['users']][0] ?? '', array('R','X')) ||
 					$no_active_participants))
 				{
-					if (++$yielded && isset($nresults) && $yielded > $nresults)
-					{
-						return;
-					}
 					// remove event from requested multiget ids, to be able to report not found urls
 					if (!empty($this->requested_multiget_ids) && ($k = array_search($event[self::$path_attr], $this->requested_multiget_ids)) !== false)
 					{
@@ -420,7 +415,8 @@ class calendar_groupdav extends Api\CalDAV\Handler
 					'getcontenttype' => $is_jscalendar ? Api\CalDAV\JsCalendar::MIME_TYPE_JSEVENT :
 						($this->agent != 'kde' ? 'text/calendar; charset=utf-8; component=VEVENT' : 'text/calendar'),
 					'getetag' => '"'.$etag.'"',
-					'getlastmodified' => $event['modified'],
+					'displayname' => $event['title'],
+					'getlastmodified' => Api\DateTime::user2server($event['modified'], 'ts'),
 					// user and timestamp of creation or last modification of event, used in calendarserver only for shared calendars
 					'created-by' => Api\CalDAV::mkprop(Api\CalDAV::CALENDARSERVER, 'created-by',
 						$this->_created_updated_by_prop($event['creator'], $event['created'])),
@@ -452,10 +448,6 @@ class calendar_groupdav extends Api\CalDAV\Handler
 						)),
 					));
 				}*/
-				if (++$yielded && isset($nresults) && $yielded > $nresults)
-				{
-					return;
-				}
 				yield  $this->add_resource($path, $event, $props);
 			}
 		}

infolog/inc/class.infolog_groupdav.inc.php

@@ -208,7 +208,7 @@ class infolog_groupdav extends Api\CalDAV\Handler
 			$files['sync-token'] = array($this, 'get_sync_collection_token');
 			$files['sync-token-params'] = array($path, $user);
 
-			$this->sync_collection_token = null;
+			$this->sync_collection_token = $this->more_results = null;
 
 			$filter['order'] = 'info_datemodified ASC';	// return oldest modifications first
 			$filter['sync-collection'] = true;
@@ -217,15 +217,6 @@ class infolog_groupdav extends Api\CalDAV\Handler
 		if (isset($nresults))
 		{
 			$files['files'] = $this->propfind_generator($path, $filter, [], $nresults);
-
-			// hack to support limit with sync-collection report: contacts are returned in modified ASC order (oldest first)
-			// if limit is smaller than full result, return modified-1 as sync-token, so client requests next chunk incl. modified
-			// (which might contain further entries with identical modification time)
-			if ($options['root']['name'] == 'sync-collection' && $this->bo->total > $nresults)
-			{
-				--$this->sync_collection_token;
-				$files['sync-token-params'][] = true;	// tel get_sync_collection_token that we have more entries
-			}
 		}
 		else
 		{
@@ -267,6 +258,8 @@ class infolog_groupdav extends Api\CalDAV\Handler
 		{
 			if (++$yielded && isset($nresults) && $yielded > $nresults)
 			{
+				$this->sync_collection_token = Api\DateTime::user2server($resource['modified'], 'ts')-1;
+				$this->more_results = true;
 				return;
 			}
 			yield $resource;
@@ -321,16 +314,18 @@ class infolog_groupdav extends Api\CalDAV\Handler
 				{
 					unset($this->requested_multiget_ids[$k]);
 				}
+				if (++$yielded && isset($nresults) && $yielded > $nresults)
+				{
+					$this->sync_collection_token = Api\DateTime::user2server($task['info_modified'], 'ts')-1;
+					$this->more_results = true;
+					return;
+				}
 				// sync-collection report: deleted entry need to be reported without properties
 				if ($task['info_status'] == 'deleted' ||
 					// or event is reported as removed from collection, because collection owner is no longer an attendee
 					$check_responsible && $task['info_owner'] != $check_responsible &&
 						!infolog_so::is_responsible_user($task, $check_responsible))
 				{
-					if (++$yielded && isset($nresults) && $yielded > $nresults)
-					{
-						return;
-					}
 					yield ['path' => $path.urldecode($this->get_path($task))];
 					continue;
 				}
@@ -354,10 +349,6 @@ class infolog_groupdav extends Api\CalDAV\Handler
 						$props['calendar-data'] = Api\CalDAV::mkprop(Api\CalDAV::CALDAV,'calendar-data',$content);
 					}
 				}
-				if (++$yielded && isset($nresults) && $yielded > $nresults)
-				{
-					return;
-				}
 				yield $this->add_resource($path, $task, $props);
 			}
 			// Please note: $query['start'] get incremented automatically by bo->search() with number of returned rows!
@@ -367,6 +358,12 @@ class infolog_groupdav extends Api\CalDAV\Handler
 				break;
 			}
 		}
+		// sync-collection report --> return modified of last contact as sync-token
+		$sync_collection_report =  $filter['sync-collection'];
+		if ($sync_collection_report)
+		{
+			$this->sync_collection_token = $task['date_modified'];
+		}
 		// report not found multiget urls
 		if (!empty($this->requested_multiget_ids))
 		{
@@ -374,17 +371,13 @@ class infolog_groupdav extends Api\CalDAV\Handler
 			{
 				if (++$yielded && isset($nresults) && $yielded > $nresults)
 				{
+					--$this->sync_collection_token;
+					$this->more_results = true;
 					return;
 				}
 				yield ['path' => $path.$id.self::$path_extension];
 			}
 		}
-		// sync-collection report --> return modified of last contact as sync-token
-		$sync_collection_report =  $filter['sync-collection'];
-		if ($sync_collection_report)
-		{
-			$this->sync_collection_token = $task['date_modified'];
-		}
 		if ($this->debug)
 		{
 			error_log(__METHOD__."($path) took ".(microtime(true) - $starttime)." to return $yielded resources, filter[sync-collection]=$sync_collection_report, sync-token=$this->sync_collection_token");

timesheet/inc/class.timesheet_bo.inc.php

@@ -485,10 +485,17 @@ class timesheet_bo extends Api\Storage
 			}
 		}
 		if(isset($filter['ts_status']) && $filter['ts_status'] && $filter['ts_status'] != self::DELETED_STATUS)
+		{
+			if ($filter['ts_status'] !== 'all')
 			{
 				$filter['ts_status'] = $this->get_sub_status($filter['ts_status']);
 			}
 			else
+			{
+				unset($filter['ts_status']);
+			}
+		}
+		else
 		{
 			$filter[] = '(ts_status ' . ($filter['ts_status'] == self::DELETED_STATUS ? '=' : '!= ') . self::DELETED_STATUS .
 				($filter['ts_status'] == self::DELETED_STATUS ? '' : ' OR ts_status IS NULL') . ')';
@@ -1112,4 +1119,39 @@ class timesheet_bo extends Api\Storage
 		}
 		return null;
 	}
+
+	/**
+	 * Get a ctag (collection tag) for timesheet
+	 *
+	 * Currently implemented as maximum modification date (1 second granularity!)
+	 *
+	 * We have to include deleted entries, as otherwise the ctag will not change if an entry gets deleted!
+	 * (Only works if tracking of deleted entries / history is switched on!)
+	 *
+	 * @param int|array $user =null
+	 * @return string
+	 */
+	public function getctag($user=null)
+	{
+		$filter = array('ts_status' => 'all');	// --> use all entries incl. deleted
+		// show timesheets of a single user?
+		if ($user) $filter['ts_owner'] = $user;
+
+		$result = $this->search(array(),'ts_modified','ts_modified DESC','','',false,'AND',array(0,1),$filter);
+
+		if (!$result || !isset($result[0]['ts_modified']))
+		{
+			$ctag = 'empty';	// ctag for empty addressbook
+		}
+		else
+		{
+			// need to convert modified time back to server-time (was converted to user-time by search)
+			// as we use it direct in server-queries eg. CardDAV sync-report and to be consistent with CalDAV
+			$ctag = Api\DateTime::user2server($result[0]['ts_modified']);
+		}
+		//error_log(__METHOD__.'('.array2string($owner).') returning '.array2string($ctag));
+		return $ctag;
+	}
+
+
 }

timesheet/src/ApiHandler.php

@@ -75,31 +75,22 @@ class ApiHandler extends Api\CalDAV\Handler
 		if ($this->debug) error_log(__METHOD__."($path,".array2string($options).",,$user,$id) filter=".array2string($filter));
 
 		// rfc 6578 sync-collection report: filter for sync-token is already set in _report_filters
-		if ($options['root']['name'] == 'sync-collection')
+		if ($options['root']['name'] === 'sync-collection')
 		{
 			// callback to query sync-token, after propfind_callbacks / iterator is run and
 			// stored max. modification-time in $this->sync_collection_token
 			$files['sync-token'] = array($this, 'get_sync_collection_token');
 			$files['sync-token-params'] = array($path, $user);
 
-			$this->sync_collection_token = null;
+			$this->sync_collection_token = $this->more_results = null;
 
 			$filter['order'] = 'ts_modified ASC';	// return oldest modifications first
 			$filter['sync-collection'] = true;
 		}
 
-		if (isset($nresults))
+		if (isset($nresults) && $options['root']['name'] === 'sync-collection')
 		{
 			$files['files'] = $this->propfind_generator($path, $filter, $files['files'], (int)$nresults);
-
-			// hack to support limit with sync-collection report: timesheets are returned in modified ASC order (oldest first)
-			// if limit is smaller than full result, return modified-1 as sync-token, so client requests next chunk incl. modified
-			// (which might contain further entries with identical modification time)
-			if ($options['root']['name'] == 'sync-collection' && $this->bo->total > $nresults)
-			{
-				--$this->sync_collection_token;
-				$files['sync-token-params'][] = true;	// tell get_sync_collection_token that we have more entries
-			}
 		}
 		else
 		{
@@ -109,6 +100,16 @@ class ApiHandler extends Api\CalDAV\Handler
 		return true;
 	}
 
+	/**
+	 * Query ctag for infolog
+	 *
+	 * @return string
+	 */
+	public function getctag($path,$user)
+	{
+		return $this->bo->getctag($user);
+	}
+
 	/**
 	 * Chunk-size for DB queries of profind_generator
 	 */
@@ -136,6 +137,8 @@ class ApiHandler extends Api\CalDAV\Handler
 		{
 			if (++$yielded && isset($nresults) && $yielded > $nresults)
 			{
+				$this->sync_collection_token = Api\DateTime::user2server($resource['modified'], 'ts')-1;
+				$this->more_results = true;
 				return;
 			}
 			yield $resource;
@@ -192,13 +195,15 @@ class ApiHandler extends Api\CalDAV\Handler
 				{
 					unset($this->requested_multiget_ids[$k]);
 				}
-				// sync-collection report: deleted entry need to be reported without properties
-				if ($timesheet['ts_status'] == \timesheet_bo::DELETED_STATUS)
-				{
 				if (++$yielded && isset($nresults) && $yielded > $nresults)
 				{
+					$this->sync_collection_token = Api\DateTime::user2server($timesheet['modified'], 'ts')-1;
+					$this->more_results = true;
 					return;
 				}
+				// sync-collection report: deleted entry need to be reported without properties
+				if ($timesheet['status'] == \timesheet_bo::DELETED_STATUS)
+				{
 					yield ['path' => $path.urldecode($this->get_path($timesheet))];
 					continue;
 				}
@@ -212,10 +217,6 @@ class ApiHandler extends Api\CalDAV\Handler
 					$props['getcontentlength'] = bytes(is_array($content) ? json_encode($content) : $content);
 					$props['data'] = Api\CalDAV::mkprop(Api\CalDAV::CARDDAV, 'data', $content);
 				}
-				if (++$yielded && isset($nresults) && $yielded > $nresults)
-				{
-					return;
-				}
 				yield $this->add_resource($path, $timesheet, $props);
 			}
 			// sync-collection report --> return modified of last timesheet as sync-token
@@ -232,6 +233,8 @@ class ApiHandler extends Api\CalDAV\Handler
 			{
 				if (++$yielded && isset($nresults) && $yielded > $nresults)
 				{
+					--$this->sync_collection_token;
+					$this->more_results = true;
 					return;
 				}
 				yield ['path' => $path.$id.self::$path_extension];
@@ -456,7 +459,7 @@ class ApiHandler extends Api\CalDAV\Handler
 						$parts = explode('/', $option['data']);
 						$sync_token = array_pop($parts);
 						$filters[] = 'ts_modified>'.(int)$sync_token;
-						$filters['tid'] = null;	// to return deleted entries too
+						$filters['ts_status'] = 'all';	// to return deleted entries too
 					}
 					break;
 				case 'sync-level':