fix for bug #1838: problem with utf-8 data in all csv imports

- fgetcsv only works correct, if setlocal is called with an existing and
  correct local
- improved projectmanager method guess_local and moved it to
  common::setlocal, which takes now the charset, lang and country of the
  user into account
- csv-import also displays now the conversation done and reads usernames
  in brackets
- added some missing fields

addressbook/csv_import.php

@@ -5,7 +5,7 @@
  * @link http://www.egroupware.org
  * @author Ralf Becker <RalfBecker-AT-outdoor-training.de>
  * @package addressbook
- * @copyright (c) 2003-8 by Ralf Becker <RalfBecker-AT-outdoor-training.de>
+ * @copyright (c) 2003-9 by Ralf Becker <RalfBecker-AT-outdoor-training.de>
  * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
  * @version $Id$
  */
@@ -34,6 +34,11 @@ if ($_POST['cancel'])
 	@unlink($csvfile);
 	$GLOBALS['egw']->redirect_link('/addressbook/index.php');
 }
+if (isset($_POST['charset']))
+{
+	// we have to set the local, to fix eg. utf-8 imports, as fgetcsv requires it!
+	common::setlocale(LC_CTYPE,$_POST['charset']);
+}
 $GLOBALS['egw_info']['flags']['app_header'] = lang('Import CSV-File into Addressbook');
 $GLOBALS['egw']->common->egw_header();
 
@@ -266,7 +271,7 @@ switch($_POST['action'])
 		$GLOBALS['egw']->preferences->add('addressbook','cvs_import',$defaults);
 		$GLOBALS['egw']->preferences->save_repository(True);
 
-		$log = "<table border=1>\n\t<tr><td>#</td>\n";
+		$log = '<table border="1" style="border: 1px dotted black; border-collapse: collapse;">'."\n\t<tr><td>#</td>\n";
 
 		foreach($addr_fields as $csv_idx => $addr)
 		{	// convert $_POST['trans'][$csv_idx] into array of pattern => value
@@ -315,7 +320,7 @@ switch($_POST['action'])
 
 			$log .= "\t</tr><tr><td>".($start+$anz)."</td>\n";
 
-			$values = array();
+			$values = $orig = array();
 			foreach($addr_fields as $csv_idx => $addr)
 			{
 				//echo "<p>$csv: $addr".($_POST['trans'][$csv] ? ': '.$_POST['trans'][$csv] : '')."</p>";
@@ -360,9 +365,7 @@ switch($_POST['action'])
 						}
 					}
 				}
-				$values[$addr] = $val;
+				$values[$addr] = $orig[$addr] = $val;
-
-				$log .= "\t\t<td>$val</td>\n";
 			}
 			$empty = !count($values);
 			// convert the category name to an id
@@ -387,7 +390,8 @@ switch($_POST['action'])
 			{
 				if (isset($values[$user]) && !is_numeric($user))
 				{
-					$values[$user] = $GLOBALS['egw']->accounts->name2id($values[$user]);
+					if (preg_match('/\[([^\]]+)\]/',$values[$user],$matches)) $values[$user] = $matches[1];
+					$values[$user] = $GLOBALS['egw']->accounts->name2id($values[$user],'account_lid',$user=='owner'?null:'u');
 				}
 			}
 			if (!in_array('owner',$addr_fields) || !$values['owner'])
@@ -415,6 +419,12 @@ switch($_POST['action'])
 				$rvalue=$GLOBALS['egw']->contacts->save($values);
 				//echo "<p>adding: ".print_r($values,true)."</p>\n";
 			}
+			// display read and interpreted results, so the user can check it
+			foreach($addr_fields as $name)
+			{
+				$log .= "\t\t<td>".($orig[$name] != $values[$name] ? htmlspecialchars($orig[$name]).' --> ' : '').
+					htmlspecialchars($values[$name])."</td>\n";
+			}
 		}
 		$log .= "\t</tr>\n</table>\n";
 

calendar/csv_import.php

@@ -5,7 +5,7 @@
  * @link http://www.egroupware.org
  * @author Ralf Becker <RalfBecker-AT-outdoor-training.de>
  * @package calendar
- * @copyright (c) 2003-8 by Ralf Becker <RalfBecker-AT-outdoor-training.de>
+ * @copyright (c) 2003-9 by Ralf Becker <RalfBecker-AT-outdoor-training.de>
  * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
  * @version $Id$
  */
@@ -32,7 +32,12 @@ else
 if ($_POST['cancel'])
 {
 	@unlink($csvfile);
-	$GLOBALS['egw']->redirect_link('/admin/index.php');
+	$GLOBALS['egw']->redirect_link('/calendar/index.php');
+}
+if (isset($_POST['charset']))
+{
+	// we have to set the local, to fix eg. utf-8 imports, as fgetcsv requires it!
+	common::setlocale(LC_CTYPE,$_POST['charset']);
 }
 $GLOBALS['egw_info']['flags']['app_header'] = $GLOBALS['egw_info']['apps']['calendar']['title'].' - '.lang('Import CSV-File');
 $cal = new calendar_boupdate(true);
@@ -171,6 +176,7 @@ case 'download':
 		'public'	=> 'Access: 1=public, 0=private',
 		'owner'		=> 'Owner: int(11) user-id/-name',
 		'modified'	=> 'Modification date',
+		'modifier'	=> 'Modification user',
 		'non_blocking' => '0=Event blocks time, 1=Event creates no conflicts',
 		'uid'       => 'Unique Id, allows multiple import to update identical entries',
 //		'recur_type'=> 'Type of recuring event',
@@ -325,14 +331,13 @@ case 'import':
 				$replaces .= ($replaces != '' ? $PSep : '') . $pattern . $ASep . $replace;
 			}
 			$trans[$csv_idx] = $values;
-		} /*else
+		}
-			unset( $trans[$csv_idx] );*/
-
 		$log .= "\t\t<td><b>$info</b></td>\n";
 	}
 	if (!in_array('public',$cal_fields))	// autocreate public access if not set by user
 	{
 		$log .= "\t\t<td><b>isPublic</b></td>\n";
+		$cal_fields[] = 'public';
 	}
 /*	if (!in_array('recur_type',$cal_fields))	// autocreate single event if not set by user
 	{
@@ -358,7 +363,7 @@ case 'import':
 
 		$log .= "\t<tr>\n\t\t<td>".($start+$anz)."</td>\n";
 
-		$values = array();
+		$values = $orig = array();
 		foreach($cal_fields as $csv_idx => $info)
 		{
 			//echo "<p>$csv: $info".($trans[$csv] ? ': '.$trans[$csv] : '')."</p>";
@@ -392,27 +397,24 @@ case 'import':
 					}
 				}
 			}
-			$values[$info] = $val;
+			$values[$info] = $orig[$info] = $val;
-
-			$log .= "\t\t<td>$val</td>\n";
 		}
-		if (!in_array('public',$cal_fields))
+		if (!isset($values['public']))
 		{
 			$values['public'] = 1;	// public access if not set by user
-			$log .= "\t\t<td>".$values['public']."</td>\n";
 		}
-		if (!in_array('recur_type',$cal_fields))
+		if (!isset($values['recur_type']))
 		{
 			$values['recur_type'] = MCAL_RECUR_NONE;	// single event if not set by user
-//				$log .= "\t\t<td>".$values['recure_type']."</td>\n";
 		}
 
-		if(!$_POST['debug'] && count($values))	// dont import empty contacts
+		if(count($values))	// dont import empty contacts
 		{
 			//echo "values=<pre>".print_r($values,True)."</pre>\n";
-			if (!is_numeric($values['owner']))
+			foreach(array('owner','modifier') as $name)
 			{
-				$values['owner'] = $GLOBALS['egw']->accounts->name2id($values['owner']);
+				if (preg_match('/\[([^\]]+)\]/',$values[$name],$matches)) $values[$name] = $matches[1];
+				if (!is_numeric($values[$name])) $values[$name] = $GLOBALS['egw']->accounts->name2id($values[$name],'account_lid','u');
 			}
 			if (!$values['owner'] || !$GLOBALS['egw']->accounts->exists($values['owner']))
 			{
@@ -452,8 +454,9 @@ case 'import':
 				list($part,$status) = explode('=',$part_status);
 				$valid_status = array('U'=>'U','u'=>'U','A'=>'A','a'=>'A','R'=>'R','r'=>'R','T'=>'T','t'=>'T');
 				$status = isset($valid_status[$status]) ? $valid_status[$status] : 'U';
-				if ($GLOBALS['egw']->accounts->exists($part))
+				if (!is_numeric($part))
 				{
+					if (preg_match('/\[([^\]]+)\]/',$part,$matches)) $part = $matches[1];
 					$part = $GLOBALS['egw']->accounts->name2id($part);
 				}
 				if ($part && is_numeric($part))
@@ -484,9 +487,8 @@ case 'import':
 			}
 			$action = $values['id'] ? 'updating' : 'adding';
 			//echo $action.'<pre>'.print_r($values,True)."</pre>\n";
-			$cal_id = $cal->update($values,true,!$values['modified'],$is_admin);	// ignoring conflicts and ACL (for admins) on import
+			if (!$_POST['debug'] &&
-
+				($cal_id = $cal->update($values,true,!$values['modified'],$is_admin)))	// ignoring conflicts and ACL (for admins) on import
-			if ($cal_id)
 			{
 				foreach(array(
 					'addressbook:'.$values['addr_id'],
@@ -501,6 +503,14 @@ case 'import':
 					}
 				}
 			}
+			// display read and interpreted results, so the user can check it
+			foreach($cal_fields as $name)
+			{
+				$log .= "\t\t<td>".($orig[$name] != $values[$name] ? htmlspecialchars($orig[$name]).' --> ' : '').
+					htmlspecialchars($values[$name])."</td>\n";
+			}
+			if(!$_POST['debug'] && count($values))	// dont import empty contacts
+			{
 				$log .= "\t\t".'<td align="center">'.($cal_id ? $action." cal_id=$cal_id" : 'Error '.$action)."</td>\n\t</tr>\n";
 			}
 			else
@@ -508,6 +518,7 @@ case 'import':
 				$log .= "\t\t".'<td align="center">only test</td>'."\n\t</tr>\n";
 			}
 		}
+	}
 	$log .= "</table>\n";
 
 	$GLOBALS['egw']->template->set_var('anz_imported',($_POST['debug'] ?

infolog/csv_import.php

@@ -5,7 +5,7 @@
  * @link http://www.egroupware.org
  * @author Ralf Becker <RalfBecker-AT-outdoor-training.de>
  * @package infolog
- * @copyright (c) 2003-8 by Ralf Becker <RalfBecker-AT-outdoor-training.de>
+ * @copyright (c) 2003-9 by Ralf Becker <RalfBecker-AT-outdoor-training.de>
  * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
  * @version $Id$
  */
@@ -36,7 +36,12 @@ else
 if ($_POST['cancel'])
 {
 	@unlink($csvfile);
-	$GLOBALS['egw']->redirect_link('/admin/index.php');
+	$GLOBALS['egw']->redirect_link('/infolog/index.php');
+}
+if (isset($_POST['charset']))
+{
+	// we have to set the local, to fix eg. utf-8 imports, as fgetcsv requires it!
+	common::setlocale(LC_CTYPE,$_POST['charset']);
 }
 $GLOBALS['egw_info']['flags']['app_header'] = lang('InfoLog - Import CSV-File');
 $GLOBALS['egw']->common->egw_header();
@@ -365,7 +370,7 @@ case 'import':
 	$GLOBALS['egw']->preferences->add('infolog','cvs_import',$defaults);
 	$GLOBALS['egw']->preferences->save_repository(True);
 
-	$log = "<table border=1>\n\t<tr><td>#</td>\n";
+	$log = '<table border="1" style="border: 1px dotted black; border-collapse: collapse;">'."\n\t<tr><td>#</td>\n";
 
 	foreach($info_fields as $csv_idx => $info)
 	{	// convert $trans[$csv_idx] into array of pattern => value
@@ -412,7 +417,7 @@ case 'import':
 
 		$log .= "\t</tr><tr><td>".($start+$anz)."</td>\n";
 
-		$values = array();
+		$values = $orig = array();
 		foreach($info_fields as $csv_idx => $info)
 		{
 			//echo "<p>$csv: $info".($trans[$csv] ? ': '.$trans[$csv] : '')."</p>";
@@ -446,9 +451,7 @@ case 'import':
 					}
 				}
 			}
-			$values[$info] = $val;
+			$values[$info] = $orig[$info] = $val;
-
-			$log .= "\t\t<td>$val</td>\n";
 		}
 		$empty = !count($values);
 
@@ -470,9 +473,13 @@ case 'import':
 		if (!isset($values['datemodified'])) $values['datemodified'] = $values['startdate'];
 
 		// convert user-names to user-id's
-		if (isset($values['owner']) && !is_numeric($values['owner']))
+		foreach(array('owner','modifier') as $user)
 		{
-			$values['owner'] = $GLOBALS['egw']->accounts->name2id($values['owner']);
+			if (isset($values[$user]) && !is_numeric($values[$user]))
+			{
+				if (preg_match('/\[([^\]]+)\]/',$values[$user],$matches)) $values[$user] = $matches[1];
+				$values[$user] = $GLOBALS['egw']->accounts->name2id($values[$user],'account_lid',$user=='owner'?null:'u');
+			}
 		}
 		if (isset($values['responsible']))
 		{
@@ -480,6 +487,7 @@ case 'import':
 			$values['responsible'] = array();
 			foreach(split('[,;]',$responsible) as $user)
 			{
+				if (preg_match('/\[([^\]]+)\]/',$user,$matches)) $user = $matches[1];
 				if ($user && !is_numeric($user)) $user = $GLOBALS['egw']->accounts->name2id($user);
 				if ($user) $values['responsible'][] = $user;
 			}
@@ -533,6 +541,12 @@ case 'import':
 				}
 			}
 		}
+		// display read and interpreted results, so the user can check it
+		foreach($info_fields as $name)
+		{
+			$log .= "\t\t<td>".($orig[$name] != $values[$name] ? htmlspecialchars($orig[$name]).' --> ' : '').
+				htmlspecialchars($values[$name])."</td>\n";
+		}
 	}
 	$log .= "\t</tr>\n</table>\n";
 

phpgwapi/inc/class.common.inc.php

@@ -24,6 +24,47 @@ class common
 	static $debug_info; // An array with debugging info from the API
 	static $found_files;
 
+	/**
+	 * Try to guess and set a locale supported by the server, with fallback to 'en_EN' and 'C'
+	 *
+	 * This method uses the language and nationalty set in the users common prefs.
+	 *
+	 * @param $category=LC_ALL category to set, see setlocal function
+	 * @param $charset=null default system charset
+	 * @return string the local (or best estimate) set
+	 */
+	static function setlocale($category=LC_ALL,$charset=null)
+	{
+		$lang = $GLOBALS['egw_info']['user']['preferences']['common']['lang'];
+		$country = $GLOBALS['egw_info']['user']['preferences']['common']['country'];
+
+		if (strlen($lang) == 2)
+		{
+			$country_from_lang = strtoupper($lang);
+		}
+		else
+		{
+			list($lang,$country_from_lang) = explode('-',$lang);
+			$country_from_lang = strtoupper($country_from_lang);
+		}
+		if (is_null($charset)) $charset = $GLOBALS['egw']->translation->charset();
+
+		foreach(array(
+			$lang.'_'.$country,
+			$lang.'_'.$country_from_lang,
+			$lang,
+			'en_EN',
+			'de_DE',	// this works with utf-8, en_EN@utf-8 does NOT!
+			'C',
+		) as $local)
+		{
+			if (($ret = setlocale($category,$local.'@'.$charset))) return $ret;
+			if (($ret = setlocale($category,$local))) return $ret;
+		}
+		error_log(__METHOD__."($category,$charset) lang=$lang, country=$country, country_from_lang=$country_from_lang: Could not set local!");
+		return false;	// should not happen, as the 'C' local should at least be available everywhere
+	}
+
 	/**
 	 * Compares two Version strings and return 1 if str2 is newest (bigger version number) than str1
 	 *