From d5b44b8e56966d2614a705e442fe256fc77ee119 Mon Sep 17 00:00:00 2001 From: Ralf Becker Date: Wed, 6 Apr 2011 12:59:52 +0000 Subject: [PATCH] * CalDAV: improved performance of ctag generation using only a single and quick DB query, compared to multiple queries plus one for each recurring event --- calendar/inc/class.calendar_bo.inc.php | 185 ++++++++++++++----- calendar/inc/class.calendar_groupdav.inc.php | 80 +------- calendar/inc/class.calendar_so.inc.php | 70 ++++++- 3 files changed, 208 insertions(+), 127 deletions(-) diff --git a/calendar/inc/class.calendar_bo.inc.php b/calendar/inc/class.calendar_bo.inc.php index 8e603ca187..1634ced11e 100644 --- a/calendar/inc/class.calendar_bo.inc.php +++ b/calendar/inc/class.calendar_bo.inc.php @@ -297,61 +297,22 @@ class calendar_bo } /** - * Searches / lists calendar entries, including repeating ones + * Resolve users to add memberships for users and members for groups * - * @param array $params array with the following keys - * start date startdate of the search/list, defaults to today - * end date enddate of the search/list, defaults to start + one day - * users int|array integer user-id or array of user-id's to use, defaults to the current user - * cat_id int|array category-id or array of cat-id's (incl. all sub-categories), default 0 = all - * filter string all (not rejected), accepted, unknown, tentative, rejected or hideprivate - * query string pattern so search for, if unset or empty all matching entries are returned (no search) - * Please Note: a search never returns repeating events more then once AND does not honor start+end date !!! - * daywise boolean on True it returns an array with YYYYMMDD strings as keys and an array with events - * (events spanning multiple days are returned each day again (!)) otherwise it returns one array with - * the events (default), not honored in a search ==> always returns an array of events! - * date_format string date-formats: 'ts'=timestamp (default), 'array'=array, or string with format for date - * offset boolean|int false (default) to return all entries or integer offset to return only a limited result - * enum_recuring boolean if true or not set (default) or daywise is set, each recurence of a recuring events is returned, - * otherwise the original recuring event (with the first start- + enddate) is returned - * num_rows int number of entries to return, default or if 0, max_entries from the prefs - * order column-names plus optional DESC|ASC separted by comma - * ignore_acl if set and true no check_perms for a general EGW_ACL_READ grants is performed - * enum_groups boolean if set and true, group-members will be added as participants with status 'G' - * cols string|array columns to select, if set an iterator will be returned - * append string to append to the query, eg. GROUP BY - * cfs array if set, query given custom fields or all for empty array, none are returned, if not set (default) - * @return iterator|array|boolean array of events or array with YYYYMMDD strings / array of events pairs (depending on $daywise param) - * or false if there are no read-grants from _any_ of the requested users or iterator/recordset if cols are given + * @param int|array $_users + * @param boolean $no_enum_groups=true + * @param boolean $ignore_acl=false + * @return array of user-ids */ - function &search($params) + private function resolve_users($_users, $no_enum_groups=true, $ignore_acl=false) { - $params_in = $params; - - unset($params['sql_filter']); // dont allow to set it via UI or xmlrpc - - // check if any resource wants to hook into - foreach($this->resources as $app => $data) + if (!is_array($_users)) { - if (isset($data['search_filter'])) - { - $params = ExecMethod($data['search_filter'],$params); - } - } - - if (!isset($params['users']) || !$params['users'] || - count($params['users']) == 1 && isset($params['users'][0]) && !$params['users'][0]) // null or '' casted to an array - { - // for a search use all account you have read grants from - $params['users'] = $params['query'] ? array_keys($this->grants) : $this->user; - } - if (!is_array($params['users'])) - { - $params['users'] = $params['users'] ? array($params['users']) : array(); + $_users = $_users ? array($_users) : array(); } // only query calendars of users, we have READ-grants from $users = array(); - foreach($params['users'] as $user) + foreach($_users as $user) { $user = trim($user); if ($params['ignore_acl'] || $this->check_perms(EGW_ACL_READ|EGW_ACL_READ_FOR_PARTICIPANTS|EGW_ACL_FREEBUSY,0,$user)) @@ -402,6 +363,62 @@ class calendar_bo } } } + return $users; + } + + /** + * Searches / lists calendar entries, including repeating ones + * + * @param array $params array with the following keys + * start date startdate of the search/list, defaults to today + * end date enddate of the search/list, defaults to start + one day + * users int|array integer user-id or array of user-id's to use, defaults to the current user + * cat_id int|array category-id or array of cat-id's (incl. all sub-categories), default 0 = all + * filter string all (not rejected), accepted, unknown, tentative, rejected or hideprivate + * query string pattern so search for, if unset or empty all matching entries are returned (no search) + * Please Note: a search never returns repeating events more then once AND does not honor start+end date !!! + * daywise boolean on True it returns an array with YYYYMMDD strings as keys and an array with events + * (events spanning multiple days are returned each day again (!)) otherwise it returns one array with + * the events (default), not honored in a search ==> always returns an array of events! + * date_format string date-formats: 'ts'=timestamp (default), 'array'=array, or string with format for date + * offset boolean|int false (default) to return all entries or integer offset to return only a limited result + * enum_recuring boolean if true or not set (default) or daywise is set, each recurence of a recuring events is returned, + * otherwise the original recuring event (with the first start- + enddate) is returned + * num_rows int number of entries to return, default or if 0, max_entries from the prefs + * order column-names plus optional DESC|ASC separted by comma + * ignore_acl if set and true no check_perms for a general EGW_ACL_READ grants is performed + * enum_groups boolean if set and true, group-members will be added as participants with status 'G' + * cols string|array columns to select, if set an iterator will be returned + * append string to append to the query, eg. GROUP BY + * cfs array if set, query given custom fields or all for empty array, none are returned, if not set (default) + * @param string $sql_filter=null sql to be and'ed into query (fully quoted), default none + * @return iterator|array|boolean array of events or array with YYYYMMDD strings / array of events pairs (depending on $daywise param) + * or false if there are no read-grants from _any_ of the requested users or iterator/recordset if cols are given + */ + function &search($params,$sql_filter=null) + { + $params_in = $params; + + $params['sql_filter'] = $sql_filter; // dont allow to set it via UI or xmlrpc + + // check if any resource wants to hook into + foreach($this->resources as $app => $data) + { + if (isset($data['search_filter'])) + { + $params = ExecMethod($data['search_filter'],$params); + } + } + + if (!isset($params['users']) || !$params['users'] || + count($params['users']) == 1 && isset($params['users'][0]) && !$params['users'][0]) // null or '' casted to an array + { + // for a search use all account you have read grants from + $params['users'] = $params['query'] ? array_keys($this->grants) : $this->user; + } + // resolve users to add memberships for users and members for groups + $users = $this->resolve_users($params['users'], $params['filter'] == 'no-enum-groups', $params['ignore_acl']); + // replace (by so not understood filter 'no-enum-groups' with 'default' filter if ($params['filter'] == 'no-enum-groups') { @@ -1850,4 +1867,76 @@ class calendar_bo return !$start['hour'] && !$start['minute'] && $end['hour'] == 23 && $end['minute'] == 59; } + + /** + * Get the etag for an entry + * + * @param array|int|string $event array with event or cal_id, or cal_id:recur_date for virtual exceptions + * @param boolean $client_share_uid_excpetions Does client understand exceptions to be included in VCALENDAR component of series master sharing its UID + * @return string|boolean string with etag or false + */ + function get_etag($entry,$client_share_uid_excpetions=true) + { + if (!is_array($entry)) + { + list($entry,$recur_date) = explode(':',$entry); + if (!$this->check_perms(EGW_ACL_FREEBUSY, $entry, 0, 'server')) return false; + $entry = $this->read($entry, $recur_date, true, 'server'); + } + $etag = $entry['id'].':'.$entry['etag']; + + // use new MAX(modification date) of egw_cal_user table (deals with virtual exceptions too) + if (isset($entry['max_user_modified'])) + { + $modified = max($entry['max_user_modified'], $entry['modified']); + } + else + { + $modified = max($this->so->max_user_modified($entry['id']), $entry['modified']); + } + $etag .= ':' . $modified; + // include exception etags into our own etag, if exceptions are included + if ($client_share_uid_excpetions && !empty($entry['uid']) && + $entry['recur_type'] != MCAL_RECUR_NONE && $entry['recur_exception']) + { + $events =& $this->search(array( + //'query' => array('cal_uid' => $entry['uid']), + 'query' => array('cal_reference' => $entry['id']), + 'filter' => 'owner', // return all possible entries + 'daywise' => false, + 'enum_recuring' => false, + 'date_format' => 'server', + 'no_total' => true, + )); + foreach($events as $k => &$recurrence) + { + if ($recurrence['reference'] && $recurrence['id'] != $entry['id']) // ignore series master + { + $etag .= ':'.$this->get_etag($recurrence); + } + } + } + //error_log(__METHOD__ . "($entry[id],$client_share_uid_excpetions) entry=".array2string($entry)." --> etag=$etag"); + return $etag; + } + + /** + * Query ctag for calendar + * + * @param int|array $user integer user-id or array of user-id's to use, defaults to the current user + * @param $filter='owner' + * @return string $filter='owner' all (not rejected), accepted, unknown, tentative, rejected or hideprivate + * @todo use MAX(modified) to query everything in one sql query, currently we do one query per event (more then the search) + */ + public function get_ctag($user,$filter='owner') + { + if ($this->debug > 1) $startime = microtime(true); + + // resolve users to add memberships for users and members for groups + $users = $this->resolve_users($user); + $ctag = $this->so->get_ctag($users, $filter == 'owner'); + + if ($this->debug > 1) error_log(__METHOD__. "($user, '$filter') = $ctag = ".date('Y-m-d H:i:s',$ctag)." took ".(microtime(true)-$startime)." secs"); + return $ctag; + } } diff --git a/calendar/inc/class.calendar_groupdav.inc.php b/calendar/inc/class.calendar_groupdav.inc.php index c8cd75d5b3..e3b52040ac 100644 --- a/calendar/inc/class.calendar_groupdav.inc.php +++ b/calendar/inc/class.calendar_groupdav.inc.php @@ -511,7 +511,7 @@ error_log(__METHOD__."($path,,".array2string($start).") filter=".array2string($f function put(&$options,$id,$user=null) { if ($this->debug) error_log(__METHOD__."($id, $user)".print_r($options,true)); - + $return_no_access = true; // as handled by importVCal anyway and allows it to set the status for participants $oldEvent = $this->_common_get_put_delete('PUT',$options,$id,$return_no_access); if (!is_null($oldEvent) && !is_array($oldEvent)) @@ -546,7 +546,7 @@ error_log(__METHOD__."($path,,".array2string($start).") filter=".array2string($f $charset = strtoupper(substr($value,1,-1)); } } - } + } } if (is_array($oldEvent)) @@ -636,7 +636,7 @@ error_log(__METHOD__."($path,,".array2string($start).") filter=".array2string($f function post(&$options,$id,$user=null) { if ($this->debug) error_log(__METHOD__."($id, $user)".print_r($options,true)); - + if (preg_match('/^METHOD:(PUBLISH|REQUEST)(\r\n|\r|\n)(.*)^BEGIN:VEVENT/ism', $options['content'])) { $handler = $this->_get_handler(); @@ -658,14 +658,14 @@ error_log(__METHOD__."($path,,".array2string($start).") filter=".array2string($f $charset = strtoupper(substr($value,1,-1)); } } - } + } } - + if (($foundEvents = $handler->search($vCalendar, null, false, $charset))) { $eventId = array_shift($foundEvents); list($eventId) = explode(':', $eventId); - + if (!($cal_id = $handler->importVCal($vCalendar, $eventId, null, false, 0, $this->principalURL, $user, $charset))) { @@ -802,38 +802,10 @@ error_log(__METHOD__."($path,,".array2string($start).") filter=".array2string($f */ public function getctag($path,$user) { - $filter = array( - 'users' => $user, - 'start' => $this->bo->now - 100*24*3600, // default one month back -30 breaks all sync recurrences - 'end' => $this->bo->now + 365*24*3600, // default one year into the future +365 - 'enum_recuring' => false, - 'daywise' => false, - 'date_format' => 'server', - 'cols' => array('egw_cal.cal_id', 'cal_start', 'cal_modified'), - ); - - if ($path == '/calendar/') - { - $filter['filter'] = 'owner'; - } - else - { - $filter['filter'] = 'default'; // not rejected - } - - $ctag = 0; - - if (($events =& $this->bo->search($filter))) - { - foreach ($events as $event) - { - $modified = max($this->bo->so->max_user_modified($event['cal_id']), $event['cal_modified']); - if ($ctag < $modified) $ctag = $modified; - } - } + $ctag = $this->bo->get_ctag($user,$path == '/calendar/' ? 'owner' : 'default'); // default = not rejected if ($this->debug > 1) error_log(__FILE__.'['.__LINE__.'] '.__METHOD__. "($path)[$user] = $ctag"); - + return 'EGw-'.$ctag.'-wGE'; } @@ -845,42 +817,8 @@ error_log(__METHOD__."($path,,".array2string($start).") filter=".array2string($f */ function get_etag($entry) { - if (!is_array($entry)) - { - if (!$this->bo->check_perms(EGW_ACL_FREEBUSY, $entry, 0, 'server')) return false; - $entry = $this->read($entry, null, true, 'server'); - } - $etag = $entry['id'].':'.$entry['etag']; + $etag = $this->bo->get_etag($entry,$this->client_shared_uid_exceptions); - // use new MAX(modification date) of egw_cal_user table (deals with virtual exceptions too) - if (isset($entry['max_user_modified'])) - { - $modified = max($entry['max_user_modified'], $entry['modified']); - } - else - { - $modified = max($this->bo->so->max_user_modified($entry['id']), $entry['modified']); - } - $etag .= ':' . $modified; - // include exception etags into our own etag, if exceptions are included - if ($this->client_shared_uid_exceptions && !empty($entry['uid']) && - $entry['recur_type'] != MCAL_RECUR_NONE && $entry['recur_exception']) - { - $events =& $this->bo->search(array( - 'query' => array('cal_uid' => $entry['uid']), - 'filter' => 'owner', // return all possible entries - 'daywise' => false, - 'enum_recuring' => false, - 'date_format' => 'server', - )); - foreach($events as $k => &$recurrence) - { - if ($recurrence['reference'] && $recurrence['id'] != $entry['id']) // ignore series master - { - $etag .= ':'.substr($this->get_etag($recurrence),4,-4); - } - } - } //error_log(__METHOD__ . "($entry[id] ($entry[etag]): $entry[title] --> etag=$etag"); return 'EGw-'.$etag.'-wGE'; } diff --git a/calendar/inc/class.calendar_so.inc.php b/calendar/inc/class.calendar_so.inc.php index 6ef88e8b95..476376aed4 100644 --- a/calendar/inc/class.calendar_so.inc.php +++ b/calendar/inc/class.calendar_so.inc.php @@ -7,7 +7,7 @@ * @author Ralf Becker * @author Christian Binder * @author Joerg Lehrke - * @copyright (c) 2005-10 by RalfBecker-At-outdoor-training.de + * @copyright (c) 2005-11 by RalfBecker-At-outdoor-training.de * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License * @version $Id$ */ @@ -259,18 +259,72 @@ class calendar_so * This includes ALL recurences of an event series * * @param int|array $ids one or multiple cal_id's + * @param booelan $return_maximum=false if true return only the maximum, even for multiple ids * @return int|array (array of) modification timestamp(s) */ - function max_user_modified($ids) + function max_user_modified($ids, $return_maximum=false) { - foreach($this->db->select($this->user_table,'cal_id,MAX(cal_user_modified) AS user_etag',array( - 'cal_id' => $ids, - ),__LINE__,__FILE__,false,'GROUP BY cal_id','calendar') as $row) + if (!is_array($ids) || count($ids) == 1) $return_maximum = true; + + if ($return_maximum) { - $etags[$row['cal_id']] = $this->db->from_timestamp($row['user_etag']); + if (($etags = $this->db->select($this->user_table,'MAX(cal_user_modified)',array( + 'cal_id' => $ids, + ),__LINE__,__FILE__,false,'','calendar')->fetchColumn())) + { + $etags = $this->db->from_timestamp($etags); + } } - //echo "

".__METHOD__.'('.array2string($ids).') = '.array2string($etags)."

\n"; - return is_array($ids) ? $etags : $etags[$ids]; + else + { + $etags = array(); + foreach($this->db->select($this->user_table,'cal_id,MAX(cal_user_modified) AS user_etag',array( + 'cal_id' => $ids, + ),__LINE__,__FILE__,false,'GROUP BY cal_id','calendar') as $row) + { + $etags[$row['cal_id']] = $this->db->from_timestamp($row['user_etag']); + } + } + //echo "

".__METHOD__.'('.array2string($ids).','.array($return_maximum).') = '.array2string($etags)."

\n"; + //error_log(__METHOD__.'('.array2string($ids).','.array2string($return_maximum).') = '.array2string($etags)); + return $etags; + } + + /** + * Get maximum modification time of events for given participants and optional owned by them + * + * This includes ALL recurences of an event series + * + * @param int|array $users one or mulitple calendar users + * @param booelan $owner_too=false if true return also events owned by given users + * @return int maximum modification timestamp + */ + function get_ctag($users, $owner_too=false) + { + $where = array( + 'cal_user_type' => 'u', + 'cal_user_id' => $users, + ); + if ($owner_too) + { + // owner can only by users, no groups + foreach($users as $key => $user) + { + if ($user < 0) unset($users[$key]); + } + $where = $this->db->expression($this->user_table, '(', $where, ' OR '). + $this->db->expression($this->cal_table, array( + 'cal_owner' => $users, + ),')'); + } + if (($data = $this->db->select($this->user_table,array( + 'MAX(cal_user_modified) AS max_user_modified', + 'MAX(cal_modified) AS max_modified', + ),$where,__LINE__,__FILE__,false,'','calendar',0,'JOIN egw_cal ON egw_cal.cal_id=egw_cal_user.cal_id')->fetch())) + { + $data = max($this->db->from_timestamp($data['max_user_modified']),$data['max_modified']); + } + return $data; } /**