From 403913f1ae5943f73a72677354db4ae2335bfbc8 Mon Sep 17 00:00:00 2001 From: Ralf Becker Date: Fri, 3 Apr 2009 12:33:43 +0000 Subject: [PATCH] fix for bug #1838: problem with utf-8 data in all csv imports - fgetcsv only works correct, if setlocal is called with an existing and correct local - improved projectmanager method guess_local and moved it to common::setlocal, which takes now the charset, lang and country of the user into account - csv-import also displays now the conversation done and reads usernames in brackets - added some missing fields --- addressbook/csv_import.php | 24 ++++++++---- calendar/csv_import.php | 63 ++++++++++++++++++------------- infolog/csv_import.php | 32 +++++++++++----- phpgwapi/inc/class.common.inc.php | 41 ++++++++++++++++++++ 4 files changed, 118 insertions(+), 42 deletions(-) diff --git a/addressbook/csv_import.php b/addressbook/csv_import.php index f3f237406d..066f4200f1 100644 --- a/addressbook/csv_import.php +++ b/addressbook/csv_import.php @@ -5,7 +5,7 @@ * @link http://www.egroupware.org * @author Ralf Becker * @package addressbook - * @copyright (c) 2003-8 by Ralf Becker + * @copyright (c) 2003-9 by Ralf Becker * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License * @version $Id$ */ @@ -34,6 +34,11 @@ if ($_POST['cancel']) @unlink($csvfile); $GLOBALS['egw']->redirect_link('/addressbook/index.php'); } +if (isset($_POST['charset'])) +{ + // we have to set the local, to fix eg. utf-8 imports, as fgetcsv requires it! + common::setlocale(LC_CTYPE,$_POST['charset']); +} $GLOBALS['egw_info']['flags']['app_header'] = lang('Import CSV-File into Addressbook'); $GLOBALS['egw']->common->egw_header(); @@ -266,7 +271,7 @@ switch($_POST['action']) $GLOBALS['egw']->preferences->add('addressbook','cvs_import',$defaults); $GLOBALS['egw']->preferences->save_repository(True); - $log = "\n\t\n"; + $log = '
#
'."\n\t\n"; foreach($addr_fields as $csv_idx => $addr) { // convert $_POST['trans'][$csv_idx] into array of pattern => value @@ -315,7 +320,7 @@ switch($_POST['action']) $log .= "\t\n"; - $values = array(); + $values = $orig = array(); foreach($addr_fields as $csv_idx => $addr) { //echo "

$csv: $addr".($_POST['trans'][$csv] ? ': '.$_POST['trans'][$csv] : '')."

"; @@ -360,9 +365,7 @@ switch($_POST['action']) } } } - $values[$addr] = $val; - - $log .= "\t\t\n"; + $values[$addr] = $orig[$addr] = $val; } $empty = !count($values); // convert the category name to an id @@ -387,7 +390,8 @@ switch($_POST['action']) { if (isset($values[$user]) && !is_numeric($user)) { - $values[$user] = $GLOBALS['egw']->accounts->name2id($values[$user]); + if (preg_match('/\[([^\]]+)\]/',$values[$user],$matches)) $values[$user] = $matches[1]; + $values[$user] = $GLOBALS['egw']->accounts->name2id($values[$user],'account_lid',$user=='owner'?null:'u'); } } if (!in_array('owner',$addr_fields) || !$values['owner']) @@ -415,6 +419,12 @@ switch($_POST['action']) $rvalue=$GLOBALS['egw']->contacts->save($values); //echo "

adding: ".print_r($values,true)."

\n"; } + // display read and interpreted results, so the user can check it + foreach($addr_fields as $name) + { + $log .= "\t\t\n"; + } } $log .= "\t\n
#
".($start+$anz)."$val".($orig[$name] != $values[$name] ? htmlspecialchars($orig[$name]).' --> ' : ''). + htmlspecialchars($values[$name])."
\n"; diff --git a/calendar/csv_import.php b/calendar/csv_import.php index d3f435cf3d..f79a560323 100644 --- a/calendar/csv_import.php +++ b/calendar/csv_import.php @@ -5,7 +5,7 @@ * @link http://www.egroupware.org * @author Ralf Becker * @package calendar - * @copyright (c) 2003-8 by Ralf Becker + * @copyright (c) 2003-9 by Ralf Becker * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License * @version $Id$ */ @@ -32,7 +32,12 @@ else if ($_POST['cancel']) { @unlink($csvfile); - $GLOBALS['egw']->redirect_link('/admin/index.php'); + $GLOBALS['egw']->redirect_link('/calendar/index.php'); +} +if (isset($_POST['charset'])) +{ + // we have to set the local, to fix eg. utf-8 imports, as fgetcsv requires it! + common::setlocale(LC_CTYPE,$_POST['charset']); } $GLOBALS['egw_info']['flags']['app_header'] = $GLOBALS['egw_info']['apps']['calendar']['title'].' - '.lang('Import CSV-File'); $cal = new calendar_boupdate(true); @@ -171,6 +176,7 @@ case 'download': 'public' => 'Access: 1=public, 0=private', 'owner' => 'Owner: int(11) user-id/-name', 'modified' => 'Modification date', + 'modifier' => 'Modification user', 'non_blocking' => '0=Event blocks time, 1=Event creates no conflicts', 'uid' => 'Unique Id, allows multiple import to update identical entries', // 'recur_type'=> 'Type of recuring event', @@ -325,16 +331,15 @@ case 'import': $replaces .= ($replaces != '' ? $PSep : '') . $pattern . $ASep . $replace; } $trans[$csv_idx] = $values; - } /*else - unset( $trans[$csv_idx] );*/ - + } $log .= "\t\t$info\n"; } if (!in_array('public',$cal_fields)) // autocreate public access if not set by user { $log .= "\t\tisPublic\n"; + $cal_fields[] = 'public'; } -/* if (!in_array('recur_type',$cal_fields)) // autocreate single event if not set by user +/* if (!in_array('recur_type',$cal_fields)) // autocreate single event if not set by user { $log .= "\t\trecureing\n"; }*/ @@ -358,7 +363,7 @@ case 'import': $log .= "\t\n\t\t".($start+$anz)."\n"; - $values = array(); + $values = $orig = array(); foreach($cal_fields as $csv_idx => $info) { //echo "

$csv: $info".($trans[$csv] ? ': '.$trans[$csv] : '')."

"; @@ -392,27 +397,24 @@ case 'import': } } } - $values[$info] = $val; - - $log .= "\t\t$val\n"; + $values[$info] = $orig[$info] = $val; } - if (!in_array('public',$cal_fields)) + if (!isset($values['public'])) { $values['public'] = 1; // public access if not set by user - $log .= "\t\t".$values['public']."\n"; } - if (!in_array('recur_type',$cal_fields)) + if (!isset($values['recur_type'])) { $values['recur_type'] = MCAL_RECUR_NONE; // single event if not set by user -// $log .= "\t\t".$values['recure_type']."\n"; } - if(!$_POST['debug'] && count($values)) // dont import empty contacts + if(count($values)) // dont import empty contacts { //echo "values=
".print_r($values,True)."
\n"; - if (!is_numeric($values['owner'])) + foreach(array('owner','modifier') as $name) { - $values['owner'] = $GLOBALS['egw']->accounts->name2id($values['owner']); + if (preg_match('/\[([^\]]+)\]/',$values[$name],$matches)) $values[$name] = $matches[1]; + if (!is_numeric($values[$name])) $values[$name] = $GLOBALS['egw']->accounts->name2id($values[$name],'account_lid','u'); } if (!$values['owner'] || !$GLOBALS['egw']->accounts->exists($values['owner'])) { @@ -452,8 +454,9 @@ case 'import': list($part,$status) = explode('=',$part_status); $valid_status = array('U'=>'U','u'=>'U','A'=>'A','a'=>'A','R'=>'R','r'=>'R','T'=>'T','t'=>'T'); $status = isset($valid_status[$status]) ? $valid_status[$status] : 'U'; - if ($GLOBALS['egw']->accounts->exists($part)) + if (!is_numeric($part)) { + if (preg_match('/\[([^\]]+)\]/',$part,$matches)) $part = $matches[1]; $part = $GLOBALS['egw']->accounts->name2id($part); } if ($part && is_numeric($part)) @@ -484,9 +487,8 @@ case 'import': } $action = $values['id'] ? 'updating' : 'adding'; //echo $action.'
'.print_r($values,True)."
\n"; - $cal_id = $cal->update($values,true,!$values['modified'],$is_admin); // ignoring conflicts and ACL (for admins) on import - - if ($cal_id) + if (!$_POST['debug'] && + ($cal_id = $cal->update($values,true,!$values['modified'],$is_admin))) // ignoring conflicts and ACL (for admins) on import { foreach(array( 'addressbook:'.$values['addr_id'], @@ -501,11 +503,20 @@ case 'import': } } } - $log .= "\t\t".''.($cal_id ? $action." cal_id=$cal_id" : 'Error '.$action)."\n\t\n"; - } - else - { - $log .= "\t\t".'only test'."\n\t\n"; + // display read and interpreted results, so the user can check it + foreach($cal_fields as $name) + { + $log .= "\t\t".($orig[$name] != $values[$name] ? htmlspecialchars($orig[$name]).' --> ' : ''). + htmlspecialchars($values[$name])."\n"; + } + if(!$_POST['debug'] && count($values)) // dont import empty contacts + { + $log .= "\t\t".''.($cal_id ? $action." cal_id=$cal_id" : 'Error '.$action)."\n\t\n"; + } + else + { + $log .= "\t\t".'only test'."\n\t\n"; + } } } $log .= "\n"; diff --git a/infolog/csv_import.php b/infolog/csv_import.php index 92617a3d31..30110ac94d 100644 --- a/infolog/csv_import.php +++ b/infolog/csv_import.php @@ -5,7 +5,7 @@ * @link http://www.egroupware.org * @author Ralf Becker * @package infolog - * @copyright (c) 2003-8 by Ralf Becker + * @copyright (c) 2003-9 by Ralf Becker * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License * @version $Id$ */ @@ -36,7 +36,12 @@ else if ($_POST['cancel']) { @unlink($csvfile); - $GLOBALS['egw']->redirect_link('/admin/index.php'); + $GLOBALS['egw']->redirect_link('/infolog/index.php'); +} +if (isset($_POST['charset'])) +{ + // we have to set the local, to fix eg. utf-8 imports, as fgetcsv requires it! + common::setlocale(LC_CTYPE,$_POST['charset']); } $GLOBALS['egw_info']['flags']['app_header'] = lang('InfoLog - Import CSV-File'); $GLOBALS['egw']->common->egw_header(); @@ -365,7 +370,7 @@ case 'import': $GLOBALS['egw']->preferences->add('infolog','cvs_import',$defaults); $GLOBALS['egw']->preferences->save_repository(True); - $log = "\n\t\n"; + $log = '
#
'."\n\t\n"; foreach($info_fields as $csv_idx => $info) { // convert $trans[$csv_idx] into array of pattern => value @@ -412,7 +417,7 @@ case 'import': $log .= "\t\n"; - $values = array(); + $values = $orig = array(); foreach($info_fields as $csv_idx => $info) { //echo "

$csv: $info".($trans[$csv] ? ': '.$trans[$csv] : '')."

"; @@ -446,9 +451,7 @@ case 'import': } } } - $values[$info] = $val; - - $log .= "\t\t\n"; + $values[$info] = $orig[$info] = $val; } $empty = !count($values); @@ -470,9 +473,13 @@ case 'import': if (!isset($values['datemodified'])) $values['datemodified'] = $values['startdate']; // convert user-names to user-id's - if (isset($values['owner']) && !is_numeric($values['owner'])) + foreach(array('owner','modifier') as $user) { - $values['owner'] = $GLOBALS['egw']->accounts->name2id($values['owner']); + if (isset($values[$user]) && !is_numeric($values[$user])) + { + if (preg_match('/\[([^\]]+)\]/',$values[$user],$matches)) $values[$user] = $matches[1]; + $values[$user] = $GLOBALS['egw']->accounts->name2id($values[$user],'account_lid',$user=='owner'?null:'u'); + } } if (isset($values['responsible'])) { @@ -480,6 +487,7 @@ case 'import': $values['responsible'] = array(); foreach(split('[,;]',$responsible) as $user) { + if (preg_match('/\[([^\]]+)\]/',$user,$matches)) $user = $matches[1]; if ($user && !is_numeric($user)) $user = $GLOBALS['egw']->accounts->name2id($user); if ($user) $values['responsible'][] = $user; } @@ -533,6 +541,12 @@ case 'import': } } } + // display read and interpreted results, so the user can check it + foreach($info_fields as $name) + { + $log .= "\t\t\n"; + } } $log .= "\t\n
#
".($start+$anz)."$val".($orig[$name] != $values[$name] ? htmlspecialchars($orig[$name]).' --> ' : ''). + htmlspecialchars($values[$name])."
\n"; diff --git a/phpgwapi/inc/class.common.inc.php b/phpgwapi/inc/class.common.inc.php index fa6fc06c76..8b419ce161 100644 --- a/phpgwapi/inc/class.common.inc.php +++ b/phpgwapi/inc/class.common.inc.php @@ -46,6 +46,47 @@ var $debug_info; // An array with debugging info from the API var $found_files; + /** + * Try to guess and set a locale supported by the server, with fallback to 'en_EN' and 'C' + * + * This method uses the language and nationalty set in the users common prefs. + * + * @param $category=LC_ALL category to set, see setlocal function + * @param $charset=null default system charset + * @return string the local (or best estimate) set + */ + static function setlocale($category=LC_ALL,$charset=null) + { + $lang = $GLOBALS['egw_info']['user']['preferences']['common']['lang']; + $country = $GLOBALS['egw_info']['user']['preferences']['common']['country']; + + if (strlen($lang) == 2) + { + $country_from_lang = strtoupper($lang); + } + else + { + list($lang,$country_from_lang) = explode('-',$lang); + $country_from_lang = strtoupper($country_from_lang); + } + if (is_null($charset)) $charset = $GLOBALS['egw']->translation->charset(); + + foreach(array( + $lang.'_'.$country, + $lang.'_'.$country_from_lang, + $lang, + 'en_EN', + 'de_DE', // this works with utf-8, en_EN@utf-8 does NOT! + 'C', + ) as $local) + { + if (($ret = setlocale($category,$local.'@'.$charset))) return $ret; + if (($ret = setlocale($category,$local))) return $ret; + } + error_log(__METHOD__."($category,$charset) lang=$lang, country=$country, country_from_lang=$country_from_lang: Could not set local!"); + return false; // should not happen, as the 'C' local should at least be available everywhere + } + /** * Compares two Version strings and return 1 if str2 is newest (bigger version number) than str1 *