WIP on caching static option file and searching it client-side

Still needs file caching & passing correct URL

api/js/etemplate/Et2Select/Et2SelectReadonly.ts

@@ -11,7 +11,7 @@
 import {css, html, LitElement, repeat, TemplateResult} from "@lion/core";
 import {et2_IDetachedDOM} from "../et2_core_interfaces";
 import {Et2Widget} from "../Et2Widget/Et2Widget";
-import {StaticOptions as so} from "./StaticOptions";
+import {StaticOptions, StaticOptions as so} from "./StaticOptions";
 import {find_select_options, SelectOption} from "./FindSelectOptions";
 import {SelectAccountMixin} from "./SelectAccountMixin";
 
@@ -52,6 +52,7 @@ li {
 
 	private __select_options : SelectOption[];
 	private __value : string[];
+	private __fetchComplete : Promise<void> = null;
 
 	constructor()
 	{
@@ -61,6 +62,16 @@ li {
 		this.__value = [];
 	}
 
+	public async getUpdateComplete()
+	{
+		const result = await super.getUpdateComplete();
+		if(this.__fetchComplete)
+		{
+			await this.__fetchComplete;
+		}
+		return result;
+	}
+
 	protected find_select_options(_attrs)
 	{
 		let sel_options = find_select_options(this, _attrs['select_options']);
@@ -68,6 +79,16 @@ li {
 		{
 			this.select_options = sel_options;
 		}
+
+		// Cache options from file
+		if(_attrs.searchUrl && _attrs.searchUrl.includes(".json") && this.__fetchComplete == null)
+		{
+			this.__fetchComplete = StaticOptions.cached_from_file(this, _attrs.searchUrl).then(options =>
+			{
+				this.select_options = options;
+				this.requestUpdate();
+			});
+		}
 	}
 
 	transformAttributes(_attrs)

api/js/etemplate/Et2Select/SearchMixin.ts

@@ -14,6 +14,7 @@ import {Validator} from "@lion/form-core";
 import {Et2Tag} from "./Tag/Et2Tag";
 import {SlMenuItem} from "@shoelace-style/shoelace";
 import {waitForEvent} from "@shoelace-style/shoelace/dist/internal/event";
+import {StaticOptions} from "./StaticOptions";
 
 // Otherwise import gets stripped
 let keep_import : Et2Tag;
@@ -576,6 +577,26 @@ export const Et2WithSearchMixin = <T extends Constructor<LitElement>>(superclass
 					}
 				});
 			}
+			if(this.searchEnabled)
+			{
+				for(const newValueElement of this.getValueAsArray())
+				{
+					if(this.select_options.some(o => o.value == newValueElement))
+					{
+						continue;
+					}
+
+					// Given a value we need to search for - this will add in all matches, including the one needed
+					this.remoteSearch(newValueElement, this.searchOptions).then((result : SelectOption[]) =>
+					{
+						const option = <SelectOption>result.find(o => o.value == newValueElement);
+						if(option)
+						{
+							this._selected_remote.push(option);
+						}
+					});
+				}
+			}
 		}
 
 		protected fix_bad_value()
@@ -1119,12 +1140,54 @@ export const Et2WithSearchMixin = <T extends Constructor<LitElement>>(superclass
 				return Promise.resolve();
 			}
 
-			// Fire off the query
+			// Check our URL: JSON file or URL?
-			let promise = this.remoteQuery(search, options);
+			if(this.searchUrl.includes(".json"))
+			{
+				// Get the file, search it
+				return this.jsonQuery(search, options);
+			}
+			else
+			{
+				// Fire off the query to the server
+				let promise = this.remoteQuery(search, options);
+			}
 
 			return promise;
 		}
 
+		/**
+		 * Search through a JSON file in the browser
+		 *
+		 * @param {string} search
+		 * @param {object} options
+		 * @protected
+		 */
+		protected jsonQuery(search : string, options : object)
+		{
+			// Get the file
+			const controller = new AbortController();
+			const signal = controller.signal;
+			let response_ok = false;
+			return StaticOptions.cached_from_file(this, this.searchUrl)
+				.then(options =>
+				{
+					// Filter the options
+					const lower_search = search.toLowerCase();
+					const filtered = options.filter(option =>
+					{
+						return option.label.toLowerCase().includes(lower_search) || option.value.includes(search)
+					});
+					// Add the matches
+					this.processRemoteResults(filtered);
+					return filtered;
+				})
+				.catch((_err) =>
+				{
+					this.egw().message(_err.statusText || this.searchUrl, "error");
+					return [];
+				});
+		}
+
 		/**
 		 * Actually query the server.
 		 *
@@ -1152,6 +1215,7 @@ export const Et2WithSearchMixin = <T extends Constructor<LitElement>>(superclass
 				{query: search, ...sendOptions}), [search, sendOptions]).then((result) =>
 			{
 				this.processRemoteResults(result);
+				return result;
 			});
 		}
 

api/js/etemplate/Et2Select/StaticOptions.ts

@@ -9,7 +9,7 @@
  */
 import {sprintf} from "../../egw_action/egw_action_common";
 import {Et2SelectReadonly} from "./Et2SelectReadonly";
-import {find_select_options, SelectOption} from "./FindSelectOptions";
+import {cleanSelectOptions, find_select_options, SelectOption} from "./FindSelectOptions";
 import {Et2Select, Et2SelectNumber, Et2WidgetWithSelect} from "./Et2Select";
 
 export type Et2SelectWidgets = Et2Select | Et2WidgetWithSelect | Et2SelectReadonly;
@@ -198,6 +198,36 @@ export const StaticOptions = new class StaticOptionsType
 		}
 	}
 
+	cached_from_file(widget, file) : Promise<SelectOption[]>
+	{
+		const cache_owner = widget.egw().getCache('Et2Select');
+		let cache = cache_owner[file];
+		if(typeof cache === 'undefined')
+		{
+			cache_owner[file] = cache = widget.egw().window.fetch(file)
+				.then((response) =>
+				{
+					// Get the options
+					if(!response.ok)
+					{
+						throw response;
+					}
+					return response.json();
+				})
+				.then(options =>
+				{
+					// Need to clean the options because file may be key=>value, may have option list, may be mixed
+					cache_owner[file] = cleanSelectOptions(options) ?? [];
+					return cache_owner[file];
+				});
+		}
+		else if(cache && typeof cache.then === "undefined")
+		{
+			return Promise.resolve(cache);
+		}
+		return cache;
+	}
+
 	priority(widget : Et2SelectWidgets) : SelectOption[]
 	{
 		return [

api/js/etemplate/et2_extension_customfields.ts

@@ -614,9 +614,11 @@ export class et2_customfields_list extends et2_valueWidget implements et2_IDetac
 		{
 			attrs.multiple = true;
 		}
-		// select_options are now send from server-side incl. ones defined via a file in EGroupware root
+		if(field.values && field.values["@"])
-		attrs.tags = field.tags;
+		{
-
+			// Options are in a list stored in a file
+			attrs.searchUrl = this.egw().webserverUrl + '/webdav.php' + field.values["@"];
+		}
 		return true;
 	}
 	_setup_select_account( field_name, field, attrs)

api/js/etemplate/et2_extension_nextmatch.ts

@@ -4190,12 +4190,17 @@ export class et2_nextmatch_customfields extends et2_customfields_list implements
 				{
 					field.values.splice(field.values.findIndex((i) => i.value == ''), 1);
 				}
+				let attrs = {
+					id: cf_id,
+					emptyLabel: field.label
+				};
+				if(field.values["@"])
+				{
+					attrs.searchUrl = this.egw().webserverUrl + '/webdav.php' + field.values["@"];
+				}
 				widget = loadWebComponent(
 					field.type == 'select-account' ? 'et2-nextmatch-header-account' : "et2-nextmatch-header-filter",
-					{
+					attrs,
-						id: cf_id,
-						empty_label: field.label
-					},
 					this
 				);
 			}
@@ -4203,7 +4208,7 @@ export class et2_nextmatch_customfields extends et2_customfields_list implements
 			{
 				widget = loadWebComponent("et2-nextmatch-header-entry", {
 					id: cf_id,
-					only_app: field.type,
+					onlyApp: field.type,
 					placeholder: field.label
 				}, this);
 			}

api/src/Etemplate/Widget/Customfields.php

@@ -244,6 +244,11 @@ class Customfields extends Transformer
 		{
 			if (!empty($data['values']))
 			{
+				// Full URL for options from file
+				if(!empty($data['values']['@']))
+				{
+					$fields[$data['name']]['values']['@'] = Api\Framework::link(Api\Vfs::download_url($data['values']['@']));
+				}
 				Select::fix_encoded_options($data['values']);
 			}
 		}
@@ -283,11 +288,14 @@ class Customfields extends Transformer
 	protected function _widget($fname, array $field)
 	{
 		static $link_types = null;
-		if (!isset($link_types)) $link_types = Api\Link::app_list ();
+		if(!isset($link_types))
+		{
+			$link_types = Api\Link::app_list();
+		}
 
 		$type = $field['type'];
 		// Link-tos needs to change from appname to link-to
-		if (!empty($link_types[$field['type']]))
+		if(!empty($link_types[$field['type']]))
 		{
 			if($type == 'filemanager')
 			{
@@ -355,21 +363,30 @@ class Customfields extends Transformer
 			case 'radio':
 				if (!empty($field['values']) && count($field['values']) == 1 && isset($field['values']['@']))
 				{
-					$field['values'] = Api\Storage\Customfields::get_options_from_file($field['values']['@']);
+					if(substr($type, 0, 7) !== 'select')
+					{
+						// Other widgets need the options, and select needs them for validation
+						$field['values'] = Api\Storage\Customfields::get_options_from_file($field['values']['@']);;
+					}
+					else
+					{
+						// Pass on no options, we do it directly through the file
+						$field['values'] = [];
+					}
 				}
 				// keep extra values set by app code, eg. addressbook advanced search
-				if (!empty(self::$request->sel_options[self::$prefix.$fname]) && is_array(self::$request->sel_options[self::$prefix.$fname]))
+				if(!empty(self::$request->sel_options[self::$prefix . $fname]) && is_array(self::$request->sel_options[self::$prefix . $fname]))
 				{
-					self::$request->sel_options[self::$prefix.$fname] += (array)$field['values'];
+					self::$request->sel_options[self::$prefix . $fname] += (array)$field['values'];
 				}
 				else
 				{
-					self::$request->sel_options[self::$prefix.$fname] = $field['values'];
+					self::$request->sel_options[self::$prefix . $fname] = $field['values'];
 				}
 				//error_log(__METHOD__."('$fname', ".array2string($field).") request->sel_options['".self::$prefix.$fname."']=".array2string(self::$request->sel_options[$this->id]));
 				// to keep order of numeric values, we have to explicit run fix_encoded_options, as sel_options are already encoded
-				$options = self::$request->sel_options[self::$prefix.$fname];
+				$options = self::$request->sel_options[self::$prefix . $fname];
-				if (is_array($options))
+				if(is_array($options))
 				{
 					Select::fix_encoded_options($options);
 					self::$request->sel_options[self::$prefix . $fname] = $options;
@@ -471,18 +488,44 @@ class Customfields extends Transformer
 				// run validation method of widget implementing this custom field
 				$widget = $this->_widget($fname, $field_settings);
 				// widget has no validate method, eg. is only displaying stuff --> nothing to validate
-				if (!method_exists($widget, 'validate')) continue;
+				if(!method_exists($widget, 'validate'))
+				{
+					continue;
+				}
+
+				// Selects only - do a local search through the file in the browser instead of sending all the options
+				// Here we need the options to validate
+				if($widget instanceof Select && !empty($field_settings['values']['@']))
+				{
+					$options = Api\Storage\Customfields::get_options_from_file($field_settings['values']['@']);
+					// keep extra values set by app code, eg. addressbook advanced search
+					if(!empty(self::$request->sel_options[self::$prefix . $fname]) && is_array(self::$request->sel_options[self::$prefix . $fname]))
+					{
+						self::$request->sel_options[$widget->id] += (array)$options;
+					}
+					else
+					{
+						self::$request->sel_options[$widget->id] = $options;
+					}
+				}
+
 				$widget->validate($form_name != self::GLOBAL_ID ? $form_name : $cname, $expand, $content, $validated);
-				$field_name = $this->id[0] == self::$prefix && $customfields[substr($this->id,strlen($this->attrs['prefix']))] ? $this->id : self::form_name($form_name != self::GLOBAL_ID ? $form_name : $cname, $field);
+				$field_name = $this->id[0] == self::$prefix && $customfields[substr($this->id, strlen($this->attrs['prefix']))] ? $this->id : self::form_name($form_name != self::GLOBAL_ID ? $form_name : $cname, $field);
 				$valid =& self::get_array($validated, $field_name, true);
 
 				// Arrays are not valid, but leave filemanager alone, we'll catch it
 				// when saving.  This allows files for new entries.
-				if (is_array($valid) && $field_settings['type'] !== 'filemanager') $valid = implode(',', $valid);
+				if(is_array($valid) && $field_settings['type'] !== 'filemanager')
+				{
+					$valid = implode(',', $valid);
+				}
 
 				// NULL is valid for most fields, but not custom fields due to backend handling
 				// See so_sql_cf->save()
-				if (is_null($valid)) $valid = false;
+				if(is_null($valid))
+				{
+					$valid = false;
+				}
 				//error_log(__METHOD__."() $field_name: ".array2string($value).' --> '.array2string($valid));
 			}
 		}

api/src/Storage/Customfields.php

@@ -272,16 +272,23 @@ class Customfields implements \IteratorAggregate
 	/**
 	 * Read the options of a 'select' or 'radio' custom field from a file
 	 *
-	 * For security reasons it has to be a php file setting one variable called options,
+	 * For security reasons it has to be a json file containing an array of options
-	 * (to not display it to anonymously by the webserver).
+	 * The contents of the file has to be either an object with value:label pairs, eg:
-	 * The $options var has to be an array with value => label pairs, eg:
 	 *
-	 * <?php
+	 * {
-	 * $options = array(
+	 *      'a': 'Option A',
-	 *      'a' => 'Option A',
+	 *      'b': 'Option B',
-	 *      'b' => 'Option B',
+	 *      'c': 'Option C',
-	 *      'c' => 'Option C',
+	 * }
-	 * );
+	 *
+	 * or an array of options with at least value and label, eg:
+	 *  [
+	 *        {value: 'a', label: 'Option A'},
+	 *        {value: 'b', label: 'Option B'},
+	 *        {value: 'c', label: 'Option C'}
+	 *  ]
+	 * See SelectOption type in api/js/etemplate/Et2Select/FindSelectOption.ts for full options.
+	 * Of the two, the array is preferred over value:label pairs for performance reasons.
 	 *
 	 * @param string $file file name inside the eGW server root, either relative to it or absolute
 	 * @return array in case of an error we return a single option with the message