From 239793470b2e92e3a7ce0f8d9d7a04a972089e73 Mon Sep 17 00:00:00 2001 From: Ralf Becker Date: Wed, 14 Apr 2010 10:19:41 +0000 Subject: [PATCH] adding an iterator for iCal files to minimize memory footprint on import of huge iCal files (not yet used for iCal data supplied as string, eg. from SyncML, as calendar_ical::importVCal uses count() and array access to returned components/events, and not just looping over it via foreach) --- calendar/inc/class.calendar_ical.inc.php | 131 +++--- calendar/inc/class.calendar_uiforms.inc.php | 12 +- phpgwapi/inc/class.egw_ical_iterator.inc.php | 439 +++++++++++++++++++ 3 files changed, 529 insertions(+), 53 deletions(-) create mode 100644 phpgwapi/inc/class.egw_ical_iterator.inc.php diff --git a/calendar/inc/class.calendar_ical.inc.php b/calendar/inc/class.calendar_ical.inc.php index eefe039f58..3ad3cc18aa 100644 --- a/calendar/inc/class.calendar_ical.inc.php +++ b/calendar/inc/class.calendar_ical.inc.php @@ -127,11 +127,11 @@ class calendar_ical extends calendar_boupdate /** * user preference: Use this timezone for import from and export to device * - * @var mixed * === false => use event's TZ * === null => export in UTC * string => device TZ * + * @var string|boolean */ var $tzid = null; @@ -980,6 +980,13 @@ class calendar_ical extends calendar_boupdate return $time->format('Ymd\THis'); } + /** + * Number of events imported in last call to importVCal + * + * @var int + */ + var $events_imported; + /** * Import an iCal * @@ -997,12 +1004,15 @@ class calendar_ical extends calendar_boupdate */ function importVCal($_vcalData, $cal_id=-1, $etag=null, $merge=false, $recur_date=0, $principalURL='', $user=null, $charset=null) { + $this->events_imported = 0; + if (!is_array($this->supportedFields)) $this->setSupportedFields(); if (!($events = $this->icaltoegw($_vcalData, $principalURL, $charset))) { return false; } + if (!is_array($events)) $cal_id = -1; // just to be sure, as iterator does NOT allow array access (eg. $events[0]) if ($cal_id > 0) { @@ -1036,6 +1046,8 @@ class calendar_ical extends calendar_boupdate } foreach ($events as $event) { + ++$this->events_imported; + if ($this->so->isWholeDay($event)) $event['whole_day'] = true; if (is_array($event['category'])) { @@ -1648,6 +1660,10 @@ class calendar_ical extends calendar_boupdate array2string($event_info['stored_event'])."\n",3,$this->logfile); } } + if (is_resource($_vcalData)) + { + date_default_timezone_set($GLOBALS['egw_info']['server']['server_timezone']); + } return $return_id; } @@ -2020,11 +2036,11 @@ class calendar_ical extends calendar_boupdate /** * Convert vCalendar data in EGw events * - * @param string $_vcalData + * @param string|resource $_vcalData * @param string $principalURL='' Used for CalDAV imports * @param string $charset The encoding charset for $text. Defaults to * utf-8 for new format, iso-8859-1 for old format. - * @return array|boolean events on success, false on failure + * @return Iterator|array|boolean Iterator if resource given or array of events on success, false on failure */ function icaltoegw($_vcalData, $principalURL='', $charset=null) { @@ -2034,8 +2050,6 @@ class calendar_ical extends calendar_boupdate array2string($_vcalData)."\n",3,$this->logfile); } - $events = array(); - if ($this->tzid) { $tzid = $this->tzid; @@ -2047,6 +2061,14 @@ class calendar_ical extends calendar_boupdate date_default_timezone_set($tzid); + if (!is_array($this->supportedFields)) $this->setSupportedFields(); + + // we use egw_ical_iterator only on resources, as calling importVCal() accesses single events like an array (eg. $events[0]) + if (is_resource($_vcalData)) + { + return new egw_ical_iterator($_vcalData,'VCALENDAR',$charset,array($this,'_ical2egw_callback'),array($tzid,$principalURL)); + } + $events = array(); $vcal = new Horde_iCalendar; if (!$vcal->parsevCalendar($_vcalData, 'VCALENDAR', $charset)) { @@ -2062,60 +2084,69 @@ class calendar_ical extends calendar_boupdate return false; } $version = $vcal->getAttribute('VERSION'); - if (!is_array($this->supportedFields)) $this->setSupportedFields(); - foreach ($vcal->getComponents() as $component) + foreach ($vcal->getComponents() as $n => $component) { - if (is_a($component, 'Horde_iCalendar_vevent')) + if (($event = $this->_ical2egw_callback($component,$tzid,$principalURL))) { - if (($event = $this->vevent2egw($component, $version, $this->supportedFields, $principalURL))) - { - //common adjustments - if ($this->productManufacturer == '' && $this->productName == '' - && !empty($event['recur_enddate'])) - { - // syncevolution needs an adjusted recur_enddate - $event['recur_enddate'] = (int)$event['recur_enddate'] + 86400; - } - if ($event['recur_type'] != MCAL_RECUR_NONE) - { - // No reference or RECURRENCE-ID for the series master - $event['reference'] = $event['recurrence'] = 0; - } - - // handle the alarms - $alarms = $event['alarm']; - foreach ($component->getComponents() as $valarm) - { - if (is_a($valarm, 'Horde_iCalendar_valarm')) - { - $this->valarm2egw($alarms, $valarm); - } - } - $event['alarm'] = $alarms; - if ($this->tzid || empty($event['tzid'])) - { - $event['tzid'] = $tzid; - } - $events[] = $event; - } - } - else - { - if ($this->log) - { - error_log(__FILE__.'['.__LINE__.'] '.__METHOD__.'()' . - get_class($component)." found\n",3,$this->logfile); - } } } - date_default_timezone_set($GLOBALS['egw_info']['server']['server_timezone']); return $events; } + /** + * Callback for egw_ical_iterator to convert Horde_iCalendar_vevent to EGw event array + * + * @param Horde_iCalendar $component + * @param string $tzid timezone + * @param string $principalURL='' Used for CalDAV imports + * @return array|boolean event array or false if $component is no Horde_iCalendar_vevent + */ + function _ical2egw_callback(Horde_iCalendar $component,$tzid,$principalURL='') + { + //unset($component->_container); _debug_array($component); + + if (!is_a($component, 'Horde_iCalendar_vevent') || + !($event = $this->vevent2egw($component, $component->getAttribute('VERSION'), $this->supportedFields, $principalURL))) + { + if ($this->log) + { + error_log(__FILE__.'['.__LINE__.'] '.__METHOD__.'() '.get_class($component)." found\n",3,$this->logfile); + } + return false; + } + //common adjustments + if ($this->productManufacturer == '' && $this->productName == '' && !empty($event['recur_enddate'])) + { + // syncevolution needs an adjusted recur_enddate + $event['recur_enddate'] = (int)$event['recur_enddate'] + 86400; + } + if ($event['recur_type'] != MCAL_RECUR_NONE) + { + // No reference or RECURRENCE-ID for the series master + $event['reference'] = $event['recurrence'] = 0; + } + + // handle the alarms + $alarms = $event['alarm']; + foreach ($component->getComponents() as $valarm) + { + if (is_a($valarm, 'Horde_iCalendar_valarm')) + { + $this->valarm2egw($alarms, $valarm); + } + } + $event['alarm'] = $alarms; + if ($this->tzid || empty($event['tzid'])) + { + $event['tzid'] = $tzid; + } + return $event; + } + /** * Parse a VEVENT * @@ -2138,7 +2169,8 @@ class calendar_ical extends calendar_boupdate return false; } - if (isset($GLOBALS['egw_info']['user']['preferences']['syncml']['minimum_uid_length'])) { + if (!empty($GLOBALS['egw_info']['user']['preferences']['syncml']['minimum_uid_length'])) + { $minimum_uid_length = $GLOBALS['egw_info']['user']['preferences']['syncml']['minimum_uid_length']; } else @@ -2780,7 +2812,6 @@ class calendar_ical extends calendar_boupdate $event['modified'] = $attributes['value']; } } - // check if the entry is a birthday // this field is only set from NOKIA clients $agendaEntryType = $component->getAttribute('X-EPOCAGENDAENTRYTYPE'); diff --git a/calendar/inc/class.calendar_uiforms.inc.php b/calendar/inc/class.calendar_uiforms.inc.php index 3086131594..8cb38be201 100644 --- a/calendar/inc/class.calendar_uiforms.inc.php +++ b/calendar/inc/class.calendar_uiforms.inc.php @@ -1642,14 +1642,20 @@ class calendar_uiforms extends calendar_ui { if (is_array($content['ical_file']) && is_uploaded_file($content['ical_file']['tmp_name'])) { - if (!ExecMethod('calendar.calendar_ical.importVCal',file_get_contents($content['ical_file']['tmp_name']))) + @set_time_limit(0); // try switching execution time limit off + $start = microtime(true); + + $calendar_ical = new calendar_ical; + if (!$calendar_ical->importVCal($f=fopen($content['ical_file']['tmp_name'],'r'))) { $msg = lang('Error: importing the iCal'); } else { - $msg = lang('iCal successful imported'); + $msg = lang('iCal successful imported').' '.lang('(%1 events in %2 seconds)', + $calendar_ical->events_imported,number_format(microtime(true)-$start,1)); } + if ($f) fclose($f); } else { @@ -1660,7 +1666,7 @@ class calendar_uiforms extends calendar_ui 'msg' => $msg, ); $GLOBALS['egw_info']['flags']['app_header'] = lang('calendar') . ' - ' . lang('iCal Import'); - $etpl = CreateObject('etemplate.etemplate','calendar.import'); + $etpl = new etemplate('calendar.import'); $etpl->exec('calendar.calendar_uiforms.import',$content); } diff --git a/phpgwapi/inc/class.egw_ical_iterator.inc.php b/phpgwapi/inc/class.egw_ical_iterator.inc.php new file mode 100644 index 0000000000..ccef127155 --- /dev/null +++ b/phpgwapi/inc/class.egw_ical_iterator.inc.php @@ -0,0 +1,439 @@ + + * @copyright (c) 2010 by Ralf Becker + * @version $Id$ + */ + +// required for tests at the end of this file (run if file called directly) +if (isset($_SERVER['SCRIPT_FILENAME']) && $_SERVER['SCRIPT_FILENAME'] == __FILE__) +{ + $GLOBALS['egw_info'] = array( + 'flags' => array( + 'currentapp' => 'calendar', + ), + ); + include('../../header.inc.php'); +} +require_once EGW_API_INC.'/horde/lib/core.php'; + +/** + * Iterator for iCal files + * + * try { + * $ical_file = fopen($path,'r'); + * $ical_it = new egw_ical_iterator($ical_file,'VCALENDAR'); + * } + * catch (Exception $e) + * { + * // could not open $path or no valid iCal file + * } + * foreach($ical_it as $vevent) + * { + * // do something with $vevent + * } + * fclose($ical_file) + */ +class egw_ical_iterator extends Horde_iCalendar implements Iterator +{ + /** + * File we work on + * + * @var resource + */ + protected $ical_file; + + /** + * Base name of container, eg. 'VCALENDAR', as passed to the constructor + * + * @var string + */ + protected $base; + + /** + * Does ical_file contain a container: BEGIN:$base ... END:$base + * + * @var boolean + */ + protected $container; + + /** + * Charset passed to the constructor + * + * @var string + */ + protected $charset; + + /** + * Current component, as it get's returned by current() method + * + * @var Horde_iCalendar + */ + protected $component; + + /** + * Callback to call with component in current() method, if returning false, item get's ignored + * + * @var callback + */ + protected $callback; + + /** + * Further parameters for the callback, 1. parameter is component + * + * @var array + */ + protected $callback_params = array(); + + /** + * Constructor + * + * @param string|resource $ical_file file opened for reading or string + * @param string $base='VCALENDAR' container + * @param string $charset=null + * @param callback $callback=null callback to call with component in current() method, if returning false, item get's ignored + * @param array $callback_params=array() further parameters for the callback, 1. parameter is component + */ + public function __construct($ical_file,$base='VCALENDAR',$charset=null,$callback=null,array $callback_params=array()) + { + // call parent constructor + parent::Horde_iCalendar(); + + $this->base = $base; + $this->charset = $charset; + if (is_callable($callback)) + { + $this->callback = $callback; + $this->callback_params = $callback_params; + } + if (is_string($ical_file)) + { + $GLOBALS[$name = md5(microtime(true))] =& $ical_file; + require_once(EGW_API_INC.'/class.global_stream_wrapper.inc.php'); + $this->ical_file = fopen('global://'.$name,'r'); + unset($GLOBALS[$name]); + // alternative: $this->unread_lines = explode("\n",$ical_file); return; + // uses less memory, but it can NOT rewind + + //error_log(__METHOD__."(,'$base','$charset') using global stream wrapper fopen('global://$name')=".array2string($this->ical_file)); + } + else + { + $this->ical_file = $ical_file; + } + if (!is_resource($this->ical_file)) + { + throw new egw_exception_wrong_parameter(__METHOD__.'($ical_file, ...) NO resource! $ical_file='.substr(array2string($ical_file),0,100)); + } + } + + /** + * Stack with not yet processed lines + * + * @var string + */ + protected $unread_lines = array(); + + /** + * Read and return one line from file (or line-buffer) + * + * We do NOT handle folding, that's done by Horde_iCalendar and not necessary for us as BEGIN: or END: component is never folded + * + * @return string|boolean string with line or false if end-of-file or end-of-container reached + */ + protected function read_line() + { + if ($this->unread_lines) + { + $line = array_shift($this->unread_lines); + } + elseif(feof($this->ical_file)) + { + $line = false; + } + else + { + $line = fgets($this->ical_file); + } + // check if end of container reached + if ($this->container && $line && substr($line,0,4+strlen($this->base)) === 'END:'.$this->base) + { + $this->unread_line($line); // put back end-of-container, to continue to return false + $line = false; + } + //error_log(__METHOD__."() returning ".($line === false ? 'FALSE' : "'$line'")); + + return $line; + } + + /** + * Take back on line, already read with read_line + * + * @param string $line + */ + protected function unread_line($line) + { + //error_log(__METHOD__."('$line')"); + array_unshift($this->unread_lines,$line); + } + + /** + * Return the current element + * + * @return Horde_iCalendar or whatever a given callback returns + */ + public function current() + { + //error_log(__METHOD__."() returning a ".gettype($this->component)); + if ($this->callback) + { + do { + if ($ret === false) $this->next(); + $params = $this->callback_params; + array_unshift($params,$this->component); + } + while(($ret = call_user_func_array($this->callback,$params)) === false); + + return $ret; + } + return $this->component; + } + + /** + * Return the key of the current element + * + * @return int|string + */ + public function key() + { + //error_log(__METHOD__."() returning ".$this->component->getAttribute('UID')); + return $this->component ? $this->component->getAttribute('UID') : false; + } + + /** + * Move forward to next component (called after each foreach loop) + */ + public function next() + { + unset($this->component); + + while (($line = $this->read_line()) && substr($line,0,6) !== 'BEGIN:') + { + // ignore it + } + if ($line === false) // end-of-file or end-of-container + { + $this->component = false; + return; + } + $type = substr(trim($line),6); + + //error_log(__METHOD__."() found $type component"); + + $data = $line; + while (($line = $this->read_line()) && substr($line,0,4+strlen($type)) !== 'END:'.$type) + { + $data .= $line; + } + $data .= $line; + + $this->component = &Horde_iCalendar::newComponent($type, $this); + //error_log(__METHOD__."() this->component = Horde_iCalendar::newComponent('$type', \$this) = ".array2string($this->component)); + if ($this->component === false) + { + error_log(__METHOD__."() Horde_iCalendar::newComponent('$type', \$this) returned FALSE"); + //return PEAR::raiseError("Unable to create object for type $type"); + } + //error_log(__METHOD__."() about to call parsevCalendar('".substr($data,0,100)."...','$type','$this->charset')"); + $this->component->parsevCalendar($data, $type, $this->charset); + + // VTIMEZONE components are NOT returned, they are only processed internally + if ($type == 'VTIMEZONE') + { + $this->addComponent($this->component); + // calling ourself recursive, to set next non-VTIMEZONE component + $this->next(); + } + } + + /** + * Rewind the Iterator to the first element (called at beginning of foreach loop) + */ + public function rewind() + { + fseek($this->ical_file,0,SEEK_SET); + + // advance to begin of container + while(($line = $this->read_line()) && substr($line,0,6+strlen($this->base)) !== 'BEGIN:'.$this->base) + { + + } + // if no container start found --> use whole file (rewind) and set container marker + if (!($this->container = $line !== false)) + { + fseek($this->ical_file,0,SEEK_SET); + } + //error_log(__METHOD__."() $this->base container ".($this->container ? 'found' : 'NOT found')); + + $data = $line; + // advance to first component + while (($line = $this->read_line()) && substr($line,0,6) !== 'BEGIN:') + { + if (preg_match('/^VERSION:(\d\.\d)\s*$/ism', $line, $matches)) + { + // define the version asap + $this->setAttribute('VERSION', $matches[1]); + } + $data .= $line; + } + // fake end of container, to get it parsed by Horde code + if ($this->container) + { + $data .= "END:$this->base\n"; + //error_log(__METHOD__."() about to call this->parsevCalendar('$data','$this->base','$this->charset')"); + $this->parsevCalendar($data,$this->base,$this->charset); + } + if ($line) $this->unread_line($line); + + // advance to first element + $this->next(); + } + + /** + * Checks if current position is valid + * + * @return boolean + */ + public function valid () + { + //error_log(__METHOD__."() returning ".(is_a($this->component,'Horde_iCalendar') ? 'TRUE' : 'FALSE').' get_class($this->component)='.get_class($this->component)); + return is_a($this->component,'Horde_iCalendar'); + } +} + +// some tests run if file called directly +if (isset($_SERVER['SCRIPT_FILENAME']) && $_SERVER['SCRIPT_FILENAME'] == __FILE__) +{ + $ical_file = 'BEGIN:VCALENDAR +PRODID:-//Microsoft Corporation//Outlook 12.0 MIMEDIR//EN +VERSION:2.0 +METHOD:PUBLISH +X-CALSTART:19980101T000000 +X-WR-RELCALID:{0000002E-1BB2-8F0F-1203-47B98FEEF211} +X-WR-CALNAME:Fxlxcxtxsxxxxxxxxxxxxx +X-PRIMARY-CALENDAR:TRUE +X-OWNER;CN="Fxlxcxtxsxxxxxxx":mailto:xexixixax.xuxaxa@xxxxxxxxxxxxxxx-berli + n.de +X-MS-OLK-WKHRSTART;TZID="Westeuropäische Normalzeit":080000 +X-MS-OLK-WKHREND;TZID="Westeuropäische Normalzeit":170000 +X-MS-OLK-WKHRDAYS:MO,TU,WE,TH,FR +BEGIN:VTIMEZONE +TZID:Westeuropäische Normalzeit +BEGIN:STANDARD +DTSTART:16011028T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:16010325T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +ATTENDEE;CN=Vorstand;RSVP=TRUE:mailto:xoxsxaxd@xxxxxxxxxxxxxxx-berlin.de +ATTENDEE;CN=\'voxrxtx@xxxxxxxx.de\';RSVP=TRUE:mailto:voxrxtx@xxxxxxxx.de +ATTENDEE;CN=Pressestelle;RSVP=TRUE:mailto:xrxsxexxexlx@xxxxxxxxxxxxxxx-berl + in.de +ATTENDEE;CN="Dxuxe Nxcxxlxxx";ROLE=OPT-PARTICIPANT;RSVP=TRUE:mailto:xjxkx.x + ixkxxsxn@xxxxxxxxxxxxxxx-berlin.de +ATTENDEE;CN="Mxxxaxx Sxxäxxr";ROLE=OPT-PARTICIPANT;RSVP=TRUE:mailto:xixhxe + x.xcxaxfxr@xxxxxxxxxxxxxxx-berlin.de +CLASS:PUBLIC +CREATED:20100408T232652Z +DESCRIPTION:\n +DTEND;TZID="Westeuropäische Normalzeit":20100414T210000 +DTSTAMP:20100406T125856Z +DTSTART;TZID="Westeuropäische Normalzeit":20100414T190000 +LAST-MODIFIED:20100408T232653Z +LOCATION:Axtx Fxuxrxaxhx\, Axex-Sxrxnxxxxxxxxxxxxxx +ORGANIZER;CN="Exixaxexhxxxxxxxxxxxräxx":mailto:xxx.xxxxxxxxxxx@xxxxxxxxxxx + xxxx-berlin.de +PRIORITY:5 +RECURRENCE-ID;TZID="Westeuropäische Normalzeit":20100414T190000 +SEQUENCE:0 +SUMMARY;LANGUAGE=de:Aktualisiert: LA - mit Ramona +TRANSP:OPAQUE +UID:040000008200E00074C5B7101A82E00800000000D0AFE96CB462CA01000000000000000 + 01000000019F8AF4D13C91844AA9CE63190D3408D +X-ALT-DESC;FMTTYPE=text/html:\n\n\n\n\n\n\n\n
\n\n\n +X-MICROSOFT-CDO-BUSYSTATUS:TENTATIVE +X-MICROSOFT-CDO-IMPORTANCE:1 +X-MICROSOFT-DISALLOW-COUNTER:FALSE +X-MS-OLK-ALLOWEXTERNCHECK:TRUE +X-MS-OLK-APPTSEQTIME:20091111T085039Z +X-MS-OLK-AUTOSTARTCHECK:FALSE +X-MS-OLK-CONFTYPE:0 +END:VEVENT +BEGIN:VEVENT +CLASS:PUBLIC +CREATED:20100331T125400Z +DTEND:20100409T110000Z +DTSTAMP:20100409T123209Z +DTSTART:20100409T080000Z +LAST-MODIFIED:20100331T125400Z +PRIORITY:5 +SEQUENCE:0 +SUMMARY;LANGUAGE=de:Marissa +TRANSP:OPAQUE +UID:AAAAAEyulq85HfZCtWDOITo5tZQHABE65KS0gg5Fu6X1g2z9eWUAAAAA3BAAABE65KS0gg5 + Fu6X1g2z9eWUAAAIQ6D0AAA== +X-MICROSOFT-CDO-BUSYSTATUS:BUSY +X-MICROSOFT-CDO-IMPORTANCE:1 +X-MS-OLK-ALLOWEXTERNCHECK:TRUE +X-MS-OLK-AUTOSTARTCHECK:FALSE +X-MS-OLK-CONFTYPE:0 +END:VEVENT +BEGIN:VEVENT +CLASS:PUBLIC +CREATED:20100331T124848Z +DTEND;VALUE=DATE:20100415 +DTSTAMP:20100409T123209Z +DTSTART;VALUE=DATE:20100414 +LAST-MODIFIED:20100331T124907Z +PRIORITY:5 +SEQUENCE:0 +SUMMARY;LANGUAGE=de:MELANIE wieder da +TRANSP:TRANSPARENT +UID:AAAAAEyulq85HfZCtWDOITo5tZQHABE65KS0gg5Fu6X1g2z9eWUAAAAA3BAAABE65KS0gg5 + Fu6X1g2z9eWUAAAIQ6DsAAA== +X-MICROSOFT-CDO-BUSYSTATUS:FREE +X-MICROSOFT-CDO-IMPORTANCE:1 +X-MS-OLK-ALLOWEXTERNCHECK:TRUE +X-MS-OLK-AUTOSTARTCHECK:FALSE +X-MS-OLK-CONFTYPE:0 +END:VEVENT +END:VCALENDAR +'; + //$ical_file = fopen('/tmp/KalenderFelicitasKubala.ics'); + if (!is_resource($ical_file)) echo "
$ical_file
\n"; + //$calendar_ical = new calendar_ical(); + //$calendar_ical->setSupportedFields('file'); + $ical_it = new egw_ical_iterator($ical_file);//,'VCALENDAR','iso-8859-1',array($calendar_ical,'_ical2egw_callback'),array('Europe/Berlin')); + foreach($ical_it as $uid => $vevent) + { + echo "$uid
".print_r($vevent,true)."
\n"; + } + if (is_resource($ical_file)) fclose($ical_file); +} \ No newline at end of file