From c3e53c9d86a6d4db796ddde69ea170d23915e5ee Mon Sep 17 00:00:00 2001 From: ralf Date: Wed, 15 Feb 2023 19:50:28 +0100 Subject: [PATCH] replaced ProfindIterator now in Calendar and InfoLog too with a generator --- .../inc/class.addressbook_groupdav.inc.php | 11 +- api/src/CalDAV/PropfindIterator.php | 186 ------------------ calendar/inc/class.calendar_groupdav.inc.php | 90 +++++---- infolog/inc/class.infolog_groupdav.inc.php | 111 ++++++----- 4 files changed, 129 insertions(+), 269 deletions(-) delete mode 100644 api/src/CalDAV/PropfindIterator.php diff --git a/addressbook/inc/class.addressbook_groupdav.inc.php b/addressbook/inc/class.addressbook_groupdav.inc.php index 519303cefd..fb3502a53a 100644 --- a/addressbook/inc/class.addressbook_groupdav.inc.php +++ b/addressbook/inc/class.addressbook_groupdav.inc.php @@ -217,10 +217,10 @@ class addressbook_groupdav extends Api\CalDAV\Handler * @param array $extra extra resources like the collection itself * @param int|null $nresults option limit of number of results to report * @param boolean $report_not_found_multiget_ids=true - * @return array with "files" array with values for keys path and props + * @return Generator * @ToDo also use CHUNK_SIZE when querying lists */ - function propfind_generator($path, array &$filter, array $extra, $nresults=null, $report_not_found_multiget_ids=true) + function propfind_generator($path, array &$filter, array $extra=[], $nresults=null, $report_not_found_multiget_ids=true) { //error_log(__METHOD__."('$path', ".array2string($filter).", ".array2string($start).", $report_not_found_multiget_ids)"); $starttime = microtime(true); @@ -259,6 +259,7 @@ class addressbook_groupdav extends Api\CalDAV\Handler // stop output buffering switched on to log the response, if we should return more than 200 entries if (!empty($this->requested_multiget_ids) && ob_get_level() && count($this->requested_multiget_ids) > 200) { + $this->caldav->log("### ".count($this->requested_multiget_ids)." resources requested in multiget REPORT --> turning logging off to allow streaming of the response"); ob_end_flush(); } @@ -307,7 +308,11 @@ class addressbook_groupdav extends Api\CalDAV\Handler // sync-collection report: deleted entry need to be reported without properties if ($contact['tid'] == Api\Contacts::DELETED_TYPE) { - $files[] = array('path' => $path.urldecode($this->get_path($contact))); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield ['path' => $path.urldecode($this->get_path($contact))]; continue; } $props = array( diff --git a/api/src/CalDAV/PropfindIterator.php b/api/src/CalDAV/PropfindIterator.php deleted file mode 100644 index a0917f6d0c..0000000000 --- a/api/src/CalDAV/PropfindIterator.php +++ /dev/null @@ -1,186 +0,0 @@ - - * @copyright (c) 2007-16 by Ralf Becker - * @version $Id$ - */ - -namespace EGroupware\Api\CalDAV; - -/** - * Iterator for propfinds using propfind callback of a Handler to query results in chunks - * - * The propfind method just computes a filter and then returns an instance of this iterator instead of the files: - * - * function propfind($path,$options,&$files,$user,$id='') - * { - * $filter = array(); - * // compute filter from path, options, ... - * - * $files['files'] = new groupdav_propfind_iterator($this,$filter,$files['files']); - * - * return true; - * } - */ -class PropfindIterator implements \Iterator -{ - /** - * current path - * - * @var string - */ - protected $path; - - /** - * Handler to call for entries - * - * @var Handler - */ - protected $handler; - - /** - * Filter of propfind call - * - * @var array - */ - protected $filter; - - /** - * Extra responses to return too - * - * @var array - */ - protected $common_files; - - /** - * current chunk - * - * @var array - */ - protected $files; - - - /** - * Start value for callback - * - * @var int - */ - protected $start=0; - - /** - * Number of entries queried from callback in one call - * - */ - const CHUNK_SIZE = 500; - - /** - * Log calls via error_log() - * - * @var boolean - */ - public $debug = false; - - /** - - /** - * Constructor - * - * @param Handler $handler - * @param array $filter filter for propfind call - * @param array $files =array() extra files/responses to return too - */ - public function __construct(Handler $handler, $path, array $filter,array &$files=array()) - { - if ($this->debug) error_log(__METHOD__."('$path', ".array2string($filter).",)"); - $this->path = $path; - $this->handler = $handler; - $this->filter = $filter; - $this->files = $this->common_files = $files; - reset($this->files); - } - - /** - * Return the current element - * - * @return array - */ - public function current() - { - if ($this->debug) error_log(__METHOD__."() returning ".array2string(current($this->files))); - return current($this->files); - } - - /** - * Return the key of the current element - * - * @return int|string - */ - public function key() - { - $current = current($this->files); - - if ($this->debug) error_log(__METHOD__."() returning ".array2string($current['path'])); - return $current['path']; // we return path as key - } - - /** - * Move forward to next element (called after each foreach loop) - */ - public function next() - { - if (next($this->files) !== false) - { - if ($this->debug) error_log(__METHOD__."() returning TRUE"); - return true; - } - // check if previous query gave not equal CHUNK_SIZE entries (or the last one is a not-found entry with just path / count()===1) --> we're done - if ($this->start && (count($this->files) != self::CHUNK_SIZE || count($this->files[self::CHUNK_SIZE-1]) === 1)) - { - if ($this->debug) error_log(__METHOD__."() returning FALSE (no more entries)"); - return false; - } - // try query further files via propfind callback of handler and store result in $this->files - $this->files = $this->handler->propfind_callback($this->path,$this->filter,array($this->start,self::CHUNK_SIZE)); - if (!is_array($this->files) || !($entries = count($this->files))) - { - if ($this->debug) error_log(__METHOD__."() returning FALSE (no more entries)"); - return false; // no further entries - } - $this->start += self::CHUNK_SIZE; - reset($this->files); - - if ($this->debug) error_log(__METHOD__."() this->start=$this->start, entries=$entries, count(this->files)=".count($this->files)." returning ".array2string(current($this->files) !== false)); - - return current($this->files) !== false; - } - - /** - * Rewind the Iterator to the first element (called at beginning of foreach loop) - */ - public function rewind() - { - if ($this->debug) error_log(__METHOD__."()"); - - $this->start = 0; - $this->files = $this->common_files; - if (!$this->files) $this->next(); // otherwise valid will return false and nothing get returned - reset($this->files); - } - - /** - * Checks if current position is valid - * - * @return boolean - */ - public function valid () - { - if ($this->debug) error_log(__METHOD__."() returning ".array2string(current($this->files) !== false)); - return current($this->files) !== false; - } -} \ No newline at end of file diff --git a/calendar/inc/class.calendar_groupdav.inc.php b/calendar/inc/class.calendar_groupdav.inc.php index 8ebd6aebdc..a0d08d6bee 100644 --- a/calendar/inc/class.calendar_groupdav.inc.php +++ b/calendar/inc/class.calendar_groupdav.inc.php @@ -85,6 +85,13 @@ class calendar_groupdav extends Api\CalDAV\Handler */ static $path_attr = 'id'; + /** + * Contains IDs for multiget REPORT to be able to report missing ones + * + * @var string[] + */ + var $requested_multiget_ids; + /** * Constructor * @@ -252,10 +259,10 @@ class calendar_groupdav extends Api\CalDAV\Handler if (isset($nresults)) { unset($filter['no_total']); // we need the total! - $files['files'] = $this->propfind_callback($path, $filter, array(0, (int)$nresults)); + $files['files'] = $this->propfind_generator($path, $filter, [], (int)$nresults); // hack to support limit with sync-collection report: events are returned in modified ASC order (oldest first) - // if limit is smaller then full result, return modified-1 as sync-token, so client requests next chunk incl. modified + // if limit is smaller than full result, return modified-1 as sync-token, so client requests next chunk incl. modified // (which might contain further entries with identical modification time) if ($options['root']['name'] == 'sync-collection' && $this->bo->total > $nresults) { @@ -265,8 +272,7 @@ class calendar_groupdav extends Api\CalDAV\Handler } else { - // return iterator, calling ourself to return result in chunks - $files['files'] = new Api\CalDAV\PropfindIterator($this,$path,$filter,$files['files']); + $files['files'] = $this->propfind_generator($path, $filter, $files['files']); } if (isset($_GET['download'])) { @@ -323,48 +329,58 @@ class calendar_groupdav extends Api\CalDAV\Handler } /** - * Callback for profind interator + * Chunk-size for DB queries of profind_generator + */ + const CHUNK_SIZE = 500; + + /** + * Generator for propfind with ability to skip reporting not found ids * * @param string $path - * @param array &$filter - * @param array|boolean $start =false false=return all or array(start,num) - * @return array with "files" array with values for keys path and props + * @param array& $filter + * @param array $extra extra resources like the collection itself + * @param int|null $nresults option limit of number of results to report + * @return Generator */ - function &propfind_callback($path, array &$filter, $start=false) + function propfind_generator($path, array &$filter, array $extra=[], $nresults=null) { if ($this->debug) $starttime = microtime(true); $calendar_data = $this->caldav->prop_requested('calendar-data', Api\CalDAV::CALDAV, true); if (!is_array($calendar_data)) $calendar_data = false; // not in allprop or autoindex - $files = array(); - - if (is_array($start)) + // yield extra resources like the root itself + $yielded = 0; + foreach($extra as $resource) { - $filter['offset'] = $start[0]; - $filter['num_rows'] = $start[1]; - } - if (!empty($filter['query'][self::$path_attr])) - { - $requested_multiget_ids =& $filter['query'][self::$path_attr]; + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield $resource; } $sync_collection = $filter['sync-collection']; - $events =& $this->bo->search($filter); - - if ($events) + for($chunk=0; $events =& $this->bo->search($filter+[ + 'offset' => $chunk*self::CHUNK_SIZE, + 'num_rows' => self::CHUNK_SIZE, + ]); ++$chunk) { foreach($events as $event) { // remove event from requested multiget ids, to be able to report not found urls - if (!empty($requested_multiget_ids) && ($k = array_search($event[self::$path_attr], $requested_multiget_ids)) !== false) + if (!empty($this->requested_multiget_ids) && ($k = array_search($event[self::$path_attr], $this->requested_multiget_ids)) !== false) { - unset($requested_multiget_ids[$k]); + unset($this->requested_multiget_ids[$k]); } // sync-collection report: deleted entries need to be reported without properties, same for rejected or deleted invitations if ($sync_collection && ($event['deleted'] && !$event['cal_reference'] || in_array($event['participants'][$filter['users']][0], array('R','X')))) { - $files[] = array('path' => $path.urldecode($this->get_path($event))); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield ['path' => $path.urldecode($this->get_path($event))]; continue; } $schedule_tag = null; @@ -406,15 +422,23 @@ class calendar_groupdav extends Api\CalDAV\Handler )), )); }*/ - $files[] = $this->add_resource($path, $event, $props); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield $this->add_resource($path, $event, $props); } } // report not found multiget urls - if (!empty($requested_multiget_ids)) + if (!empty($this->requested_multiget_ids)) { - foreach($requested_multiget_ids as $id) + foreach($this->requested_multiget_ids as $id) { - $files[] = array('path' => $path.$id.self::$path_extension); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield ['path' => $path.$id.self::$path_extension]; } } // sync-collection report --> return modified of last contact as sync-token @@ -426,9 +450,8 @@ class calendar_groupdav extends Api\CalDAV\Handler if ($this->debug) { error_log(__METHOD__."($path) took ".(microtime(true) - $starttime). - ' to return '.count($files['files']).' items'); + " to return $yielded resources"); } - return $files; } /** @@ -595,14 +618,13 @@ class calendar_groupdav extends Api\CalDAV\Handler } // multiget or propfind on a given id //error_log(__FILE__ . __METHOD__ . "multiget of propfind:"); + $this->requested_multiget_ids = null; if ($options['root']['name'] == 'calendar-multiget' || $id) { // no standard time-range! unset($cal_filters['start']); unset($cal_filters['end']); - $ids = array(); - if ($id) { $cal_filters['query'][self::$path_attr] = self::$path_extension ? @@ -610,6 +632,7 @@ class calendar_groupdav extends Api\CalDAV\Handler } else // fetch all given url's { + $this->requested_multiget_ids = []; foreach($options['other'] as $option) { if ($option['name'] == 'href') @@ -617,11 +640,12 @@ class calendar_groupdav extends Api\CalDAV\Handler $parts = explode('/',$option['data']); if (($id = urldecode(array_pop($parts)))) { - $cal_filters['query'][self::$path_attr][] = self::$path_extension ? + $this->requested_multiget_ids[] = self::$path_extension ? basename($id,self::$path_extension) : $id; } } } + $cal_filters['query'][self::$path_attr] = $this->requested_multiget_ids; } if ($this->debug > 1) error_log(__FILE__ . __METHOD__ ."($options[path],...,$id) calendar-multiget: ids=".implode(',',$ids).', cal_filters='.array2string($cal_filters)); diff --git a/infolog/inc/class.infolog_groupdav.inc.php b/infolog/inc/class.infolog_groupdav.inc.php index 552dd630be..9e25163aeb 100644 --- a/infolog/inc/class.infolog_groupdav.inc.php +++ b/infolog/inc/class.infolog_groupdav.inc.php @@ -56,6 +56,13 @@ class infolog_groupdav extends Api\CalDAV\Handler */ static $path_attr = 'info_id'; + /** + * Contains IDs for multiget REPORT to be able to report missing ones + * + * @var string[] + */ + var $requested_multiget_ids; + /** * Constructor * @@ -204,10 +211,10 @@ class infolog_groupdav extends Api\CalDAV\Handler if (isset($nresults)) { - $files['files'] = $this->propfind_callback($path, $filter, array(0, (int)$nresults)); + $files['files'] = $this->propfind_generator($path, $filter, [], $nresults); // hack to support limit with sync-collection report: contacts are returned in modified ASC order (oldest first) - // if limit is smaller then full result, return modified-1 as sync-token, so client requests next chunk incl. modified + // if limit is smaller than full result, return modified-1 as sync-token, so client requests next chunk incl. modified // (which might contain further entries with identical modification time) if ($options['root']['name'] == 'sync-collection' && $this->bo->total > $nresults) { @@ -217,21 +224,26 @@ class infolog_groupdav extends Api\CalDAV\Handler } else { - // return iterator, calling ourself to return result in chunks - $files['files'] = new Api\CalDAV\PropfindIterator($this,$path,$filter,$files['files']); + $files['files'] = $this->propfind_generator($path,$filter, $files['files']); } return true; } /** - * Callback for profind interator + * Chunk-size for DB queries of profind_generator + */ + const CHUNK_SIZE = 500; + + /** + * Generator for propfind with ability to skip reporting not found ids * * @param string $path - * @param array &$filter - * @param array|boolean $start =false false=return all or array(start,num) - * @return array with "files" array with values for keys path and props + * @param array& $filter + * @param array $extra extra resources like the collection itself + * @param int|null $nresults option limit of number of results to report + * @return Generator */ - function &propfind_callback($path,array &$filter,$start=false) + function propfind_generator($path, array &$filter, array $extra=[], $nresults=null) { if ($this->debug) $starttime = microtime(true); @@ -243,6 +255,17 @@ class infolog_groupdav extends Api\CalDAV\Handler $task_filter = $filter['filter']; unset($filter['filter']); + // yield extra resources like the root itself + $yielded = 0; + foreach($extra as $resource) + { + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield $resource; + } + $order = 'info_datemodified'; $sort = 'DESC'; $matches = null; @@ -272,32 +295,19 @@ class infolog_groupdav extends Api\CalDAV\Handler $query['cols'] = array('main.info_id AS info_id', 'info_datemodified', 'info_uid', 'caldav_name', 'info_subject', 'info_status', 'info_owner'); } - if (is_array($start)) - { - $query['start'] = $offset = $start[0]; - $query['num_rows'] = $start[1]; - } - else - { - $offset = 0; - } - - if (!empty($filter[self::$path_attr])) - { - $requested_multiget_ids =& $filter[self::$path_attr]; - } - $files = array(); // ToDo: add parameter to only return id & etag - $tasks =& $this->bo->search($query); - if ($tasks && $offset == $query['start']) + for($chunk=0; ($params = $query+[ + 'start' => $chunk*self::CHUNK_SIZE, + 'num_rows' => self::CHUNK_SIZE, + ]) && ($tasks =& $this->bo->search($params)); ++$chunk) { foreach($tasks as $task) { // remove task from requested multiget ids, to be able to report not found urls - if (!empty($requested_multiget_ids) && ($k = array_search($task[self::$path_attr], $requested_multiget_ids)) !== false) + if (!empty($this->requested_multiget_ids) && ($k = array_search($task[self::$path_attr], $this->requested_multiget_ids)) !== false) { - unset($requested_multiget_ids[$k]); + unset($this->requested_multiget_ids[$k]); } // sync-collection report: deleted entry need to be reported without properties if ($task['info_status'] == 'deleted' || @@ -305,7 +315,11 @@ class infolog_groupdav extends Api\CalDAV\Handler $check_responsible && $task['info_owner'] != $check_responsible && !infolog_so::is_responsible_user($task, $check_responsible)) { - $files[] = array('path' => $path.urldecode($this->get_path($task))); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield ['path' => $path.urldecode($this->get_path($task))]; continue; } $props = array( @@ -319,15 +333,23 @@ class infolog_groupdav extends Api\CalDAV\Handler $props['getcontentlength'] = bytes($content); $props[] = Api\CalDAV::mkprop(Api\CalDAV::CALDAV,'calendar-data',$content); } - $files[] = $this->add_resource($path, $task, $props); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield $this->add_resource($path, $task, $props); } } // report not found multiget urls - if (!empty($requested_multiget_ids)) + if (!empty($this->requested_multiget_ids)) { - foreach($requested_multiget_ids as $id) + foreach($this->requested_multiget_ids as $id) { - $files[] = array('path' => $path.$id.self::$path_extension); + if (++$yielded && isset($nresults) && $yielded > $nresults) + { + return; + } + yield ['path' => $path.$id.self::$path_extension]; } } // sync-collection report --> return modified of last contact as sync-token @@ -335,17 +357,11 @@ class infolog_groupdav extends Api\CalDAV\Handler if ($sync_collection_report) { $this->sync_collection_token = $task['date_modified']; - - // hack to support limit with sync-collection report: tasks are returned in modified ASC order (oldest first) - // if limit is smaller then full result, return modified-1 as sync-token, so client requests next chunk incl. modified - // (which might contain further entries with identical modification time) - if ($start[0] == 0 && $start[1] != Api\CalDAV\PropfindIterator::CHUNK_SIZE && $this->bo->total > $start[1]) - { - --$this->sync_collection_token; - } } - if ($this->debug) error_log(__METHOD__."($path) took ".(microtime(true) - $starttime).' to return '.count($files).' items'); - return $files; + if ($this->debug) + { + error_log(__METHOD__."($path) took ".(microtime(true) - $starttime)." to return $yielded resources"); + } } /** @@ -445,10 +461,9 @@ class infolog_groupdav extends Api\CalDAV\Handler } } // multiget or propfind on a given id - //error_log(__FILE__ . __METHOD__ . "multiget of propfind:"); + $this->requested_multiget_ids = null; if ($options['root']['name'] == 'calendar-multiget' || $id) { - $ids = array(); if ($id) { $cal_filters[self::$path_attr] = self::$path_extension ? @@ -456,6 +471,7 @@ class infolog_groupdav extends Api\CalDAV\Handler } else // fetch all given url's { + $this->requested_multiget_ids = []; foreach($options['other'] as $option) { if ($option['name'] == 'href') @@ -463,13 +479,14 @@ class infolog_groupdav extends Api\CalDAV\Handler $parts = explode('/',$option['data']); if (($id = basename(urldecode(array_pop($parts))))) { - $cal_filters[self::$path_attr][] = self::$path_extension ? + $this->requested_multiget_ids[] = self::$path_extension ? basename($id,self::$path_extension) : $id; } } } + $cal_filters[self::$path_attr] = $this->requested_multiget_ids; } - if ($this->debug > 1) error_log(__METHOD__ ."($options[path],...,$id) calendar-multiget: ids=".implode(',',$ids)); + if ($this->debug > 1) error_log(__METHOD__ ."($options[path],...,$id) calendar-multiget: ids=".implode(',', $this->requested_multiget_ids)); } return true; }