From 447c8b618a8f21d37a406d8e65fcac4d6c937e02 Mon Sep 17 00:00:00 2001 From: Ralf Becker Date: Sat, 17 Oct 2009 09:13:36 +0000 Subject: [PATCH] Using an iterator to query addressbook in chunks of 100 contacts to allow to do propfinds on hugh addressbooks independent of memory_limit: - regular groupdav_handler::profind() method gets split in a method just computing a filter and a callback to run that filter on the backend - groupdav_propfind_iterator class is returned from profind method instead of an array with information about the files - iterator calls groupdav_hander::propfind_callback if there are no more entries from the previous call - constructor of groupdav_propfind_iterator allows to pass an extra array with files to return, to simplify modifying existing implementation (were eg. information about the current path, get's supplied from calling groupdav class). --- .../inc/class.addressbook_groupdav.inc.php | 46 ++++-- phpgwapi/inc/class.groupdav_handler.inc.php | 149 +++++++++++++++++- 2 files changed, 176 insertions(+), 19 deletions(-) diff --git a/addressbook/inc/class.addressbook_groupdav.inc.php b/addressbook/inc/class.addressbook_groupdav.inc.php index 518c0ce998..4b1114df5d 100644 --- a/addressbook/inc/class.addressbook_groupdav.inc.php +++ b/addressbook/inc/class.addressbook_groupdav.inc.php @@ -1,6 +1,6 @@ debug) error_log(__METHOD__."($path,".array2string($options).",,$user,$id) filter=".array2string($filter)); // check if we have to return the full calendar data or just the etag's - if (!($address_data = $options['props'] == 'all' && $options['root']['ns'] == groupdav::CARDDAV) && is_array($options['props'])) + if (!($filter['address_data'] = $options['props'] == 'all' && $options['root']['ns'] == groupdav::CARDDAV) && is_array($options['props'])) { foreach($options['props'] as $prop) { if ($prop['name'] == 'address-data') { - $address_data = true; + $filter['address_data'] = true; break; } } } - if ($address_data) + // return iterator, calling ourself to return result in chunks + $files['files'] = new groupdav_propfind_iterator($this,$filter,$files['files']); + + return true; + } + + /** + * Callback for profind interator + * + * @param array $filter + * @param array|boolean $start=false false=return all or array(start,num) + * @return array with "files" array with values for keys path and props + */ + function &propfind_callback(array $filter,$start=false) + { + $starttime = microtime(true); + + if (($address_data = $filter['address_data'])) { $handler = self::_get_handler(); } - $start = false; - //$start = array(0,7000); // limit the number of contacts to return + unset($filter['address_data']); + + $files = array(); // we query etag and modified, as LDAP does not have the strong sql etag if (($contacts =& $this->bo->search(array(),$address_data ? false : array('id','uid','etag','modified'),'contact_id','','',False,'AND',$start,$filter))) { - //$icount= 0; foreach($contacts as &$contact) { $props = array( @@ -125,10 +144,7 @@ class addressbook_groupdav extends groupdav_handler ); if ($address_data) { - //$sta = microtime(true); $content = $handler->getVCard($contact,$this->charset,false); - //$en = microtime(true) - $sta; - //error_log("getVCard took : $en"); $props[] = HTTP_WebDAV_Server::mkprop('getcontentlength',bytes($content)); $props[] = HTTP_WebDAV_Server::mkprop(groupdav::CARDDAV,'address-data',$content); } @@ -136,14 +152,14 @@ class addressbook_groupdav extends groupdav_handler { $props[] = HTTP_WebDAV_Server::mkprop('getcontentlength', ''); // expensive to calculate and no CalDAV client uses it } - $files['files'][] = array( + $files[] = array( 'path' => self::get_path($contact), 'props' => $props, ); } } - if ($this->debug) error_log(__METHOD__."($path) took ".(microtime(true) - $starttime).' to return '.count($files['files']).' items'); - return true; + if ($this->debug) error_log(__METHOD__.'('.array2string($filter).','.array2string($start).") took ".(microtime(true) - $starttime).' to return '.count($files).' items'); + return $files; } /** diff --git a/phpgwapi/inc/class.groupdav_handler.inc.php b/phpgwapi/inc/class.groupdav_handler.inc.php index c55c5e28d5..429f408aa5 100644 --- a/phpgwapi/inc/class.groupdav_handler.inc.php +++ b/phpgwapi/inc/class.groupdav_handler.inc.php @@ -1,18 +1,18 @@ - * @copyright (c) 2007/8 by Ralf Becker + * @copyright (c) 2007-9 by Ralf Becker * @version $Id$ */ /** - * eGroupWare: GroupDAV access: abstract baseclass for groupdav/caldav/carddav handlers + * EGroupware: GroupDAV access: abstract baseclass for groupdav/caldav/carddav handlers */ abstract class groupdav_handler { @@ -86,7 +86,7 @@ abstract class groupdav_handler #if (!is_null($debug)) $this->debug = $debug = 3; $this->base_uri = is_null($base_uri) ? $base_uri : $_SERVER['SCRIPT_NAME']; $this->agent = self::get_agent(); - + $this->translation =& $GLOBALS['egw']->translation; $this->egw_charset = $this->translation->charset(); } @@ -102,6 +102,16 @@ abstract class groupdav_handler */ abstract function propfind($path,$options,&$files,$user); + /** + * Propfind callback, if interator is used + * + * @param array $filter + * @param array|boolean $start false=return all or array(start,num) + * @param int &$total + * @return array with "files" array with values for keys path and props + */ + function &propfind_callback(array $filter,$start,&$total) { } + /** * Handle get request for an applications entry * @@ -306,3 +316,134 @@ abstract class groupdav_handler } } +/** + * Iterator for propfinds using propfind callback of a groupdav_handler to query results in chunks + * + * The propfind method just computes a filter and then returns an instance of this iterator instead of the files: + * + * function propfind($path,$options,&$files,$user,$id='') + * { + * $filter = array(); + * // compute filter from path, options, ... + * + * $files['files'] = new groupdav_propfind_iterator($this,$filter,$files['files']); + * + * return true; + * } + */ +class groupdav_propfind_iterator implements Iterator +{ + /** + * Handler to call for entries + * + * @var groupdav_handler + */ + protected $handler; + + /** + * Filter of propfind call + * + * @var array + */ + protected $filter; + + /** + * Extra responses to return too + * + * @var array + */ + protected $files; + + /** + * Start value for callback + * + * @var int + */ + protected $start=0; + + /** + * Number of entries queried from callback in one call + * + */ + const CHUNK_SIZE = 100; + + /** + * Constructor + * + * @param groupdav_handler $handler + * @param array $filter filter for propfind call + * @param array $files=null extra files/responses to return too + */ + public function __construct(groupdav_handler $handler,array $filter,array &$files=null) + { + $this->handler = $handler; + $this->filter = $filter; + $this->files = $files; + reset($this->files); + } + + /** + * Return the current element + * + * @return array + */ + public function current() + { + return current($this->files); + } + + /** + * Return the key of the current element + * + * @return int|string + */ + public function key() + { + $current = $this->current(); + + return $current['path']; // we return path as key + } + + /** + * Move forward to next element (called after each foreach loop) + */ + public function next() + { + if (next($this->files) !== false) + { + return true; + } + if (!$this->handler) + { + return false; // no further entries + } + // try query further files via propfind callback of handler and store result in $this->files + $this->files = $this->handler->propfind_callback($this->filter,array($this->start,self::CHUNK_SIZE)); + $this->start += self::CHUNK_SIZE; + reset($this->files); + + if (count($this->files) < self::CHUNK_SIZE) // less entries then asked --> no further available + { + unset($this->handler); + } + return current($this->files) !== false; + } + + /** + * Rewind the Iterator to the first element (called at beginning of foreach loop) + */ + public function rewind() + { + + } + + /** + * Checks if current position is valid + * + * @return boolean + */ + public function valid () + { + return current($this->files) !== false; + } +}