Fix scheduled import from remote url

importexport/inc/class.importexport_schedule_ui.inc.php

@@ -40,7 +40,10 @@
 			if(is_array($async_list)) {
 				foreach($async_list as $id => $async) {
 					if(is_array($async['data']['errors'])) {
-						$async['data']['errors'] = implode("\n", $async['data']['errors']);
+						foreach($async['data']['errors'] as $target => $errors)
+						{
+							$async['data']['errors'] = $target . ":\n" . implode("\n", $async['data']['errors']);
+						}
 					}
 					if(is_numeric($async['data']['record_count'])) {
 						$async['data']['record_count'] = lang('%1 records processed', $async['data']['record_count']);
@@ -123,6 +126,7 @@
 				if($file_check !== true) $data['message'] .= ($data['message'] ? "\n" . $file_check : $file_check);
 			}
 
+			$data['no_delete_files'] = $data['type'] != 'import';
 			$sel_options = self::get_select_options($data);
 			$GLOBALS['egw']->js->validate_file('.','importexport','importexport');
 
@@ -147,13 +151,14 @@
 				$options['appname'] = array('' => lang('Select one')) + array_combine($apps,$apps);
 			}
 
-			if($data['appname']) {
+			$plugins = importexport_helper_functions::get_plugins($data['appname'] ? $data['appname'] : 'all', $data['type']);
-				$plugins = importexport_helper_functions::get_plugins($data['appname'], $data['type']);
+			if(is_array($plugins)) {
-				if(is_array($plugins[$data['appname']][$data['type']])) {
+				foreach($plugins as $appname => $types) {
-					foreach($plugins[$data['appname']][$data['type']] as $key => $title) {
+					if(!is_array($types[$data['type']])) continue;
+					foreach($types[$data['type']] as $key => $title) {
 						$options['plugin'][$key] = $title;
 					}
-				}
+				}	
 			}
 
 			$options['definition'] = array();
@@ -223,6 +228,7 @@
 			} else {
 				$response->addScript("xajax_doXMLHTTP('importexport.importexport_schedule_ui.ajax_get_definitions', '$appname', document.getElementById('exec[plugin]').value);");
 			}
+
 			return $response->getXML();
 		}
 
@@ -262,11 +268,33 @@
 				stream_wrapper_register(vfs_stream_wrapper::SCHEME, 'vfs_stream_wrapper', STREAM_IS_URL);
 			}
 
-			if ($data['type'] == 'import' && ($scheme == egw_vfs::SCHEME && !egw_vfs::is_readable($data['target']) ||
+			if ($data['type'] == 'import' && ($scheme == egw_vfs::SCHEME && !egw_vfs::is_readable($data['target'])))
-			    $scheme != egw_vfs::SCHEME && !is_readable($data['target'])))
 			{
 				return lang('%1 is not readable',$data['target']);
 			}
+			elseif ($data['type'] == 'import' && in_array($scheme, array('http','https')))
+			{
+				// Not supported by is_readable, try headers...
+				$options = array();
+				stream_context_set_default(array('http'=>array(
+					'method'	=> 'HEAD',
+					'ignore_errors'	=> 1
+				)));
+				$headers = get_headers($data['target'],1);
+
+				// Reset...
+				stream_context_set_default(array('http'=>array(
+					'method'	=> 'GET',
+					'ignore_errors'	=> 0
+				)));
+				// Response code has an integer key, but redirects may add more responses
+				for($i = 0; $i < count($headers); $i++)
+				{
+					if(!$headers[$i]) break;
+					if(strpos($headers[$i],'200') !== false) return true;
+				}
+				return lang('%1 is not readable',$data['target']);
+			}
 			elseif ($data['type'] == 'export' && !self::is__writable($data['target'])) {
 				return lang('%1 is not writable',$data['target']);
 			}
@@ -321,37 +349,82 @@
 			$po = new $definition->plugin;
 
 			$type = $data['type'];
-			if($resource = fopen( $data['target'], $data['type'] == 'import' ? 'r' : 'w' )) {
-				$result = $po->$type( $resource, $definition );
 
-				fclose($resource);
+			
-			} else {
+			$data['record_count'] = 0;
-				fwrite(STDERR,'importexport_schedule: ' . date('c') . ": Definition not found! \n");
+			unset($data['errors']);
+			unset($data['result']);
+
+			if(is_dir($data['target']))
+			{
+				$contents = scandir($data['target']);
+				$targets = array_diff($contents, array('.','..'));
+				foreach($targets as $key => &$target)
+				{
+					$target = $data['target'].'/'.$target;
+					
+					// Check modification time, make sure it's not currently being written
+					// Skip files modified in the last 10 seconds
+					if(filemtime($target) >= time() - 10) 
+					{
+						$data['result'][$target] = lang('Skipped');
+						unset($target[$key]);
+					}
+				}
+				unset($target); // Unset it, or it will be overwritten in loop below
+			}
+			else
+			{
+				$targets = array($data['target']);
 			}
 
-			if(method_exists($po, 'get_errors') && $po->get_errors()) {
+			foreach($targets as $target)
-				$data['errors'] = $po->get_errors();
+			{
-				fwrite(STDERR, 'importexport_schedule: ' . date('c') . ": Import errors:\n#\tError\n");
+				if($resource = @fopen( $target, $data['type'] == 'import' ? 'rb' : 'wb' )) {
-				foreach($po->get_errors() as $record => $error) {
+					$result = $po->$type( $resource, $definition );
-					fwrite(STDERR, "$record\t$error\n");
-				}
-			} else {
-				unset($data['errors']);
-			}
 
-			if($po instanceof importexport_iface_import_plugin) {
+					fclose($resource);
-				if(is_numeric($result)) {
+				} else {
-					$data['record_count'] = $result;
+					fwrite(STDERR,'importexport_schedule: ' . date('c') . ": Definition not found! \n");
 				}
-				$data['result'] = '';
+			
-				foreach($po->get_results() as $action => $count) {
+
-					$data['result'] .= "\n" . lang($action) . ": $count";
+				if(method_exists($po, 'get_errors') && $po->get_errors()) {
+					$data['errors'][$target] = $po->get_errors();
+					fwrite(STDERR, 'importexport_schedule: ' . date('c') . ": Import errors:\n#\tError\n");
+					foreach($po->get_errors() as $record => $error) {
+						fwrite(STDERR, "$record\t$error\n");
+					}
+				} else {
+					unset($data['errors'][$target]);
+				}
+
+				if($po instanceof importexport_iface_import_plugin) {
+					if(is_numeric($result)) {
+						$data['record_count'] += $result;
+					}
+					$data['result'][$target] = '';
+					foreach($po->get_results() as $action => $count) {
+						$data['result'][$target] .= "\n" . lang($action) . ": $count";
+					}
+				} else {
+					$data['result'][$target] = $result;
 				}
-			} else {
-				$data['result'] = $result;
 			}
 			$data['last_run'] = time();
 
+			// Delete file?
+			if($data['delete_files'] && $type == 'import' && !$data['errors'][$target])
+			{
+				foreach($targets as $target)
+				{
+					if(unlink($target))
+					{
+						$data['result'][$target] .= "\n..." . lang('deleted');
+					}
+				}
+			}
+
 			// Update job with results
 			$id = self::generate_id($data);
 			$async = ExecMethod('phpgwapi.asyncservice.read', $id);

importexport/setup/etemplates.inc.php

@@ -2,7 +2,7 @@
 /**
  * EGroupware - eTemplates for Application importexport
  * http://www.egroupware.org
- * generated by soetemplate::dump4setup() 2011-12-05 10:05
+ * generated by soetemplate::dump4setup() 2011-12-05 17:24
  *
  * @license http://opensource.org/licenses/gpl-license.php GPL - GNU General Public License
  * @package importexport
@@ -97,7 +97,7 @@ $templ_data[] = array('name' => 'importexport.import_dialog','template' => '','l
 display: none;
 }','modified' => '1320333891',);
 
-$templ_data[] = array('name' => 'importexport.schedule_edit','template' => '','lang' => '','group' => '0','version' => '1.9.002','data' => 'a:1:{i:0;a:4:{s:4:"type";s:4:"grid";s:4:"data";a:10:{i:0;a:2:{s:2:"h1";s:10:",!@message";s:2:"c7";s:4:"help";}i:1;a:2:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:11:"all,message";s:4:"name";s:7:"message";}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:2;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Type";}s:1:"B";a:3:{s:4:"type";s:6:"select";s:4:"name";s:4:"type";s:8:"onchange";i:1;}}i:3;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:11:"Application";}s:1:"B";a:3:{s:4:"type";s:6:"select";s:4:"name";s:7:"appname";s:8:"onchange";s:137:"xajax_doXMLHTTP(\'importexport.importexport_schedule_ui.ajax_get_plugins\', document.getElementById(form::name(\'type\')).value, this.value);";}}i:4;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Plugin";}s:1:"B";a:4:{s:4:"type";s:6:"select";s:4:"name";s:6:"plugin";s:4:"size";s:9:"Select...";s:8:"onchange";s:144:"xajax_doXMLHTTP(\'importexport.importexport_schedule_ui.ajax_get_definitions\', document.getElementById(form::name(\'appname\')).value, this.value);";}}i:5;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:10:"Definition";}s:1:"B";a:3:{s:4:"type";s:6:"select";s:4:"name";s:10:"definition";s:4:"size";s:9:"Select...";}}i:6;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Target";}s:1:"B";a:3:{s:4:"type";s:4:"text";s:4:"size";s:2:"50";s:4:"name";s:6:"target";}}i:7;a:2:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:3:"all";s:5:"label";s:83:"Target examples: vfs://default/home/user/export.csv or http://server.net/prices.csv";}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:8;a:2:{s:1:"A";a:7:{s:4:"type";s:4:"grid";s:4:"data";a:4:{i:0;a:1:{s:2:"c1";s:2:"th";}i:1;a:6:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Year";}s:1:"B";a:2:{s:4:"type";s:5:"label";s:5:"label";s:5:"Month";}s:1:"C";a:2:{s:4:"type";s:5:"label";s:5:"label";s:3:"Day";}s:1:"D";a:2:{s:4:"type";s:5:"label";s:5:"label";s:11:"Day of week";}s:1:"E";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Hour";}s:1:"F";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Minute";}}i:2;a:6:{s:1:"A";a:3:{s:4:"type";s:4:"text";s:4:"name";s:4:"year";s:4:"size";s:1:"5";}s:1:"B";a:3:{s:4:"type";s:4:"text";s:4:"name";s:5:"month";s:4:"size";s:1:"5";}s:1:"C";a:3:{s:4:"type";s:4:"text";s:4:"name";s:3:"day";s:4:"size";s:1:"5";}s:1:"D";a:4:{s:4:"type";s:4:"text";s:4:"name";s:3:"dow";s:5:"align";s:6:"center";s:4:"size";s:1:"5";}s:1:"E";a:3:{s:4:"type";s:4:"text";s:4:"name";s:4:"hour";s:4:"size";s:1:"5";}s:1:"F";a:3:{s:4:"type";s:4:"text";s:4:"name";s:3:"min";s:4:"size";s:1:"5";}}i:3;a:6:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:1:"3";s:5:"label";s:11:"(* for all)";}s:1:"B";a:1:{s:4:"type";s:5:"label";}s:1:"C";a:1:{s:4:"type";s:5:"label";}s:1:"D";a:2:{s:4:"type";s:5:"label";s:5:"label";s:12:"(0-6, 0=Sun)";}s:1:"E";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"(0-23)";}s:1:"F";a:1:{s:4:"type";s:5:"label";}}}s:4:"rows";i:3;s:4:"cols";i:6;s:4:"name";s:8:"schedule";s:4:"span";s:3:"all";s:7:"options";a:0:{}}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:9;a:2:{s:1:"A";a:5:{s:4:"type";s:4:"hbox";s:4:"size";s:1:"2";s:4:"span";s:3:"all";i:1;a:3:{s:4:"type";s:6:"button";s:5:"label";s:4:"Save";s:4:"name";s:4:"save";}i:2;a:4:{s:4:"type";s:6:"button";s:4:"name";s:6:"cancel";s:5:"label";s:6:"Cancel";s:7:"onclick";s:13:"self.close();";}}s:1:"B";a:1:{s:4:"type";s:5:"label";}}}s:4:"rows";i:9;s:4:"cols";i:2;}}','size' => '','style' => '','modified' => '1300135635',);
+$templ_data[] = array('name' => 'importexport.schedule_edit','template' => '','lang' => '','group' => '0','version' => '1.9.003','data' => 'a:1:{i:0;a:4:{s:4:"type";s:4:"grid";s:4:"data";a:11:{i:0;a:3:{s:2:"h1";s:10:",!@message";s:2:"c7";s:4:"help";s:2:"h8";s:17:",@no_delete_files";}i:1;a:2:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:11:"all,message";s:4:"name";s:7:"message";}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:2;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Type";}s:1:"B";a:3:{s:4:"type";s:6:"select";s:4:"name";s:4:"type";s:8:"onchange";i:1;}}i:3;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:11:"Application";}s:1:"B";a:3:{s:4:"type";s:6:"select";s:4:"name";s:7:"appname";s:8:"onchange";s:137:"xajax_doXMLHTTP(\'importexport.importexport_schedule_ui.ajax_get_plugins\', document.getElementById(form::name(\'type\')).value, this.value);";}}i:4;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Plugin";}s:1:"B";a:4:{s:4:"type";s:6:"select";s:4:"name";s:6:"plugin";s:4:"size";s:9:"Select...";s:8:"onchange";s:144:"xajax_doXMLHTTP(\'importexport.importexport_schedule_ui.ajax_get_definitions\', document.getElementById(form::name(\'appname\')).value, this.value);";}}i:5;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:10:"Definition";}s:1:"B";a:3:{s:4:"type";s:6:"select";s:4:"name";s:10:"definition";s:4:"size";s:9:"Select...";}}i:6;a:2:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Target";}s:1:"B";a:3:{s:4:"type";s:4:"text";s:4:"size";s:2:"50";s:4:"name";s:6:"target";}}i:7;a:2:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:3:"all";s:5:"label";s:83:"Target examples: vfs://default/home/user/export.csv or http://server.net/prices.csv";}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:8;a:2:{s:1:"A";a:4:{s:4:"type";s:8:"checkbox";s:5:"label";s:25:"Delete files after import";s:4:"name";s:12:"delete_files";s:4:"span";s:3:"all";}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:9;a:2:{s:1:"A";a:7:{s:4:"type";s:4:"grid";s:4:"data";a:4:{i:0;a:1:{s:2:"c1";s:2:"th";}i:1;a:6:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Year";}s:1:"B";a:2:{s:4:"type";s:5:"label";s:5:"label";s:5:"Month";}s:1:"C";a:2:{s:4:"type";s:5:"label";s:5:"label";s:3:"Day";}s:1:"D";a:2:{s:4:"type";s:5:"label";s:5:"label";s:11:"Day of week";}s:1:"E";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Hour";}s:1:"F";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Minute";}}i:2;a:6:{s:1:"A";a:3:{s:4:"type";s:4:"text";s:4:"name";s:4:"year";s:4:"size";s:1:"5";}s:1:"B";a:3:{s:4:"type";s:4:"text";s:4:"name";s:5:"month";s:4:"size";s:1:"5";}s:1:"C";a:3:{s:4:"type";s:4:"text";s:4:"name";s:3:"day";s:4:"size";s:1:"5";}s:1:"D";a:4:{s:4:"type";s:4:"text";s:4:"name";s:3:"dow";s:5:"align";s:6:"center";s:4:"size";s:1:"5";}s:1:"E";a:3:{s:4:"type";s:4:"text";s:4:"name";s:4:"hour";s:4:"size";s:1:"5";}s:1:"F";a:3:{s:4:"type";s:4:"text";s:4:"name";s:3:"min";s:4:"size";s:1:"5";}}i:3;a:6:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:1:"3";s:5:"label";s:11:"(* for all)";}s:1:"B";a:1:{s:4:"type";s:5:"label";}s:1:"C";a:1:{s:4:"type";s:5:"label";}s:1:"D";a:2:{s:4:"type";s:5:"label";s:5:"label";s:12:"(0-6, 0=Sun)";}s:1:"E";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"(0-23)";}s:1:"F";a:1:{s:4:"type";s:5:"label";}}}s:4:"rows";i:3;s:4:"cols";i:6;s:4:"name";s:8:"schedule";s:4:"span";s:3:"all";s:7:"options";a:0:{}}s:1:"B";a:1:{s:4:"type";s:5:"label";}}i:10;a:2:{s:1:"A";a:5:{s:4:"type";s:4:"hbox";s:4:"size";s:1:"2";s:4:"span";s:3:"all";i:1;a:3:{s:4:"type";s:6:"button";s:5:"label";s:4:"Save";s:4:"name";s:4:"save";}i:2;a:4:{s:4:"type";s:6:"button";s:4:"name";s:6:"cancel";s:5:"label";s:6:"Cancel";s:7:"onclick";s:13:"self.close();";}}s:1:"B";a:1:{s:4:"type";s:5:"label";}}}s:4:"rows";i:10;s:4:"cols";i:2;}}','size' => '','style' => '','modified' => '1323130226',);
 
 $templ_data[] = array('name' => 'importexport.schedule_index','template' => '','lang' => '','group' => '0','version' => '1.7.001','data' => 'a:1:{i:0;a:4:{s:4:"type";s:4:"grid";s:4:"data";a:4:{i:0;a:1:{s:2:"h1";s:6:",!@msg";}i:1;a:1:{s:1:"A";a:3:{s:4:"type";s:5:"label";s:4:"span";s:11:"all,message";s:4:"name";s:4:"@msg";}}i:2;a:1:{s:1:"A";a:6:{s:4:"type";s:4:"grid";s:4:"name";s:9:"scheduled";s:7:"options";a:0:{}s:4:"data";a:3:{i:0;a:2:{s:2:"c1";s:2:"th";s:2:"c2";s:4:",top";}i:1;a:8:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:5:"label";s:4:"Type";}s:1:"B";a:2:{s:4:"type";s:5:"label";s:5:"label";s:11:"Application";}s:1:"C";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Plugin";}s:1:"D";a:2:{s:4:"type";s:5:"label";s:5:"label";s:10:"Definition";}s:1:"E";a:2:{s:4:"type";s:5:"label";s:5:"label";s:6:"Target";}s:1:"F";a:2:{s:4:"type";s:5:"label";s:5:"label";s:8:"Last Run";}s:1:"G";a:4:{s:4:"type";s:4:"vbox";s:4:"size";s:6:"2,,0,0";i:1;a:2:{s:4:"type";s:5:"label";s:5:"label";s:8:"Next Run";}i:2;a:2:{s:4:"type";s:5:"label";s:5:"label";s:8:"Schedule";}}s:1:"H";a:1:{s:4:"type";s:5:"label";}}i:2;a:8:{s:1:"A";a:2:{s:4:"type";s:5:"label";s:4:"name";s:12:"${row}[type]";}s:1:"B";a:3:{s:4:"type";s:10:"select-app";s:4:"name";s:15:"${row}[appname]";s:8:"readonly";s:1:"1";}s:1:"C";a:4:{s:4:"type";s:6:"select";s:7:"no_lang";s:1:"1";s:4:"name";s:14:"${row}[plugin]";s:8:"readonly";s:1:"1";}s:1:"D";a:3:{s:4:"type";s:6:"select";s:4:"name";s:18:"${row}[definition]";s:8:"readonly";s:1:"1";}s:1:"E";a:3:{s:4:"type";s:5:"label";s:7:"no_lang";s:1:"1";s:4:"name";s:14:"${row}[target]";}s:1:"F";a:6:{s:4:"type";s:4:"vbox";s:4:"size";s:1:"4";i:1;a:3:{s:4:"type";s:9:"date-time";s:4:"name";s:16:"${row}[last_run]";s:8:"readonly";s:1:"1";}i:2;a:3:{s:4:"type";s:5:"label";s:4:"name";s:20:"${row}[record_count]";s:7:"no_lang";s:1:"1";}i:3;a:3:{s:4:"type";s:5:"label";s:4:"name";s:14:"${row}[result]";s:7:"no_lang";s:1:"1";}i:4;a:2:{s:4:"type";s:5:"label";s:4:"name";s:14:"${row}[errors]";}}s:1:"G";a:4:{s:4:"type";s:4:"vbox";s:4:"size";s:6:"2,,0,0";i:1;a:3:{s:4:"type";s:9:"date-time";s:4:"name";s:12:"${row}[next]";s:8:"readonly";s:1:"1";}i:2;a:3:{s:4:"type";s:5:"label";s:7:"no_lang";s:1:"1";s:4:"name";s:13:"${row}[times]";}}s:1:"H";a:4:{s:4:"type";s:4:"hbox";s:4:"size";s:6:"2,,0,0";i:1;a:4:{s:4:"type";s:6:"button";s:4:"size";s:4:"edit";s:4:"name";s:21:"edit[${row_cont[id]}]";s:7:"onclick";s:198:"window.open(egw::link(\'/index.php\',\'menuaction=importexport.importexport_schedule_ui.edit&id={$row_cont[id]}\'),\'_blank\',\'dependent=yes,width=600,height=450,scrollbars=yes,status=yes\'); return false;";}i:2;a:3:{s:4:"type";s:6:"button";s:4:"size";s:6:"delete";s:4:"name";s:23:"delete[{$row_cont[id]}]";}}}}s:4:"rows";i:2;s:4:"cols";i:8;}}i:3;a:1:{s:1:"A";a:4:{s:4:"type";s:4:"hbox";s:4:"size";s:1:"1";s:4:"span";s:3:"all";i:1;a:4:{s:4:"type";s:6:"button";s:5:"label";s:3:"add";s:4:"name";s:3:"add";s:7:"onclick";s:179:"window.open(egw::link(\'/index.php\',\'menuaction=importexport.importexport_schedule_ui.edit\'),\'_blank\',\'dependent=yes,width=600,height=450,scrollbars=yes,status=yes\'); return false;";}}}}s:4:"rows";i:3;s:4:"cols";i:1;}}','size' => '','style' => '','modified' => '1267034207',);