Various bug fixes in the grid, implemented prefetching, workaround for performance issues regarding update, removed _lastModification

etemplate/js/et2_core_common.js

@@ -659,3 +659,30 @@ function et2_rangeEqual(_ar1, _ar2)
 	return _ar1.top === _ar2.top && _ar1.bottom === _ar2.bottom;
 }
 
+/**
+ * Substracts _ar2 from _ar1, returns an array of new ranges.
+ */
+function et2_rangeSubstract(_ar1, _ar2)
+{
+	// Per default return the complete _ar1 range
+	var res = [_ar1];
+
+	// Check whether there is an intersection between the given ranges
+	if (et2_rangeIntersect(_ar1, _ar2))
+	{
+		res = [et2_bounds(_ar1.top, _ar2.top),
+			   et2_bounds(_ar2.bottom, _ar1.bottom)];
+	}
+
+	// Remove all zero-length ranges from the result
+	for (var i = res.length - 1; i >= 0; i--)
+	{
+		if (res[i].bottom - res[i].top <= 0)
+		{
+			res.splice(i, 1);
+		}
+	}
+
+	return res;
+}
+

etemplate/js/et2_dataview_controller.js

@@ -24,6 +24,8 @@
  */
 var ET2_DATAVIEW_FETCH_TIMEOUT = 50;
 
+var ET2_DATAVIEW_STEPSIZE = 50;
+
 /**
  * The et2_dataview_controller class is the intermediate layer between a grid
  * instance and the corresponding data source. It manages updating the grid,
@@ -47,15 +49,12 @@ var et2_dataview_controller = Class.extend({
 		// containers hashed by the "index"
 		this._indexMap = {};
 
-		// "lastModified" contains the last timestap which was returned from the
-		// server.
-		this._lastModification = null;
-
 		// Timer used for queing fetch requests
 		this._queueTimer = null;
 
-		// Array used for queing the requests
+		// Array which contains all currently queued indices in the form of
-		this._queue = [];
+		// an associative array
+		this._queue = {};
 
 		// Register the dataFetch callback
 		this._grid.setDataCallback(this._gridCallback, this);
@@ -75,9 +74,28 @@ var et2_dataview_controller = Class.extend({
 	 */
 	update: function () {
 		// Clear the fetch queue
-		this._queue = [];
+		this._queue = {};
 		this._clearTimer();
 
+		// ---------
+
+		// TODO: Actually stuff here should be done if the server responds that
+		// there at all were some changes (needs implementation of "refresh")
+
+		// Remove all rows which are outside the view range
+		this._grid.cleanup();
+
+		// Remove all index entries which are currently not displayed
+		for (var key in this._indexMap)
+		{
+			if (!this._indexMap[key].row)
+			{
+				delete this._indexMap[key];
+			}
+		}
+
+		// ---------
+
 		// Get the currently visible range from the grid
 		var range = this._grid.getIndexRange();
 
@@ -88,8 +106,7 @@ var et2_dataview_controller = Class.extend({
 		}
 
 		// Require that range from the server
-		this._queueFetch(range.top, range.bottom - range.top + 1,
+		this._queueFetch(et2_bounds(range.top, range.bottom + 1), true);
-				this._lastModification !== null, true);
 	},
 
 	/**
@@ -98,7 +115,6 @@ var et2_dataview_controller = Class.extend({
 	reset: function () {
 		// Throw away all internal mappings and reset the timestamp
 		this._indexMap = {};
-		this._lastModification = null;
 
 		// Clear the grid
 		this._grid.clear();
@@ -222,135 +238,144 @@ var et2_dataview_controller = Class.extend({
 		// Queue fetching that data range
 		if (needsData !== false)
 		{
-			this._queueFetch(needsData, _idxEnd - needsData + 1, false);
+			this._queueFetch(et2_bounds(needsData, _idxEnd + 1), false);
 		}
 	},
 
 	/**
-	 * 
+	 * The _queueFetch function is used to queue a fetch request.
+	 * TODO: Refresh is currently not used
 	 */
-	_queueFetch: function (_start, _numRows, _refresh, _immediate) {
+	_queueFetch: function (_range, _isUpdate) {
 
 		// Force immediate to be false
-//		_immediate = _immediate ? _immediate : false;
+		_isUpdate = _isUpdate ? _isUpdate : false;
-		_immediate = true;
 
-		// Push the request onto the request queue
+		// Push the requests onto the request queue
-		this._queue.push({
+		var start = Math.max(0, _range.top);
-				"start": _start,
+		var end = Math.min(this._grid.getTotalCount(), _range.bottom);
-				"num_rows": _numRows,
+		for (var i = start; i < end; i++)
-				"refresh": _refresh
+		{
-		});
+			if (typeof this._queue[i] === "undefined")
+			{
+				this._queue[i] = 1; // Stage 1 -- queued
+			}
+		}
 
 		// Start the queue timer, if this has not already been done
-		if (this._queueTimer === null && !_immediate)
+		if (this._queueTimer === null && !_isUpdate)
 		{
 			var self = this;
 			this._queueTimer = window.setTimeout(function () {
-				self._flushQueue();
+				self._flushQueue(false);
 			}, ET2_DATAVIEW_FETCH_TIMEOUT);
 		}
 
-		if (_immediate)
+		if (_isUpdate)
 		{
-			this._flushQueue();
+			this._flushQueue(true);
 		}
 	},
 
-	_flushQueue: function () {
+	/**
-
+	 * Flushes the queue.
-		function consolidateQueries(_q) {
+	 */
-			var didConsolidation = false;
+	_flushQueue: function (_isUpdate) {
-
-			var _new = [];
-			var skip = {};
-
-			for (var i = 0; i < _q.length; i++)
-			{
-				var r1 = et2_range(_q[i].start, _q[i].num_rows);
-
-				var intersected = false;
-
-				for (var j = i + 1; j < _q.length; j++)
-				{
-					if (skip[j])
-					{
-						continue;
-					}
-
-					var r2 = et2_range(_q[j].start, _q[j].num_rows);
-
-					if (et2_rangeIntersect(r1, r2))
-					{
-						var n = et2_bounds(Math.min(r1.top, r2.top),
-								Math.max(r1.botom, r2.bottom));
-						_new.push({
-							"start": n.top,
-							"num_rows": n.bottom - n.top + 1,
-							"refresh": _q[i].refresh
-						});
-						skip[i] = true;
-						skip[j] = true;
-						intersected = true;
-					}
-				}
-
-				if (!intersected)
-				{
-					_new.push(_q[i]);
-					skip[i] = true;
-				}
-			}
-
-			if (didConsolidation) {
-				return consolidateQueries(_new);
-			}
-
-			return _new;
-		}
 
 		// Clear any still existing timer
 		this._clearTimer();
 
-		// Calculate the refresh flag (refresh = false is stronger)
+		// Mark all elements in a radius of ET2_DATAVIEW_STEPSIZE
-		var refresh = true;
+		var marked = {};
-		for (var i = 0; i < this._queue.length; i++)
+		var r = _isUpdate ? 0 : Math.floor(ET2_DATAVIEW_STEPSIZE / 2);
+		var total = this._grid.getTotalCount();
+		for (var key in this._queue)
 		{
-			refresh = refresh && this._queue[i].refresh;
+			if (this._queue[key] > 1)
+				continue;
+
+			key = parseInt(key);
+
+			var b = Math.max(0, key - r);
+			var t = Math.min(key + r, total - 1);
+			var c = 0;
+			for (var i = b; i <= t && c < ET2_DATAVIEW_STEPSIZE; i ++)
+			{
+				if (typeof this._queue[i] == "undefined"
+						|| this._queue[i] === 1)
+				{
+					this._queue[i] = 2; // Stage 2 -- pending or available
+					marked[i] = true;
+					c++;
+				}
+			}
 		}
 
-		// Extend all ranges into bottom direction, initialize the queries array
+		// Create a list with start indices and counts
-		for (var i = 0; i < this._queue.length; i++)
+		var fetchList = [];
+		var entry = null;
+		var last = 0;
+
+		// Get the int keys and sort the array numeric
+		var arr = et2_arrayIntKeys(marked).sort(
+				function(a,b){return a > b ? 1 : (a == b ? 0 : -1)});
+
+		for (var i = 0; i < arr.length; i++)
 		{
-			this._queue[i].num_rows += 10;
+			if (i == 0 || arr[i] - last > 1)
-			this._queue[i].refresh = refresh;
+			{
+				if (entry)
+				{
+					fetchList.push(entry);
+				}
+				entry = {
+					"start": arr[i],
+					"count": 1
+				};
+			}
+			else
+			{
+				entry.count++;
+			}
+
+			last = arr[i];
 		}
 
-		// Consolidate all queries
+		if (entry)
-		var queries = consolidateQueries(this._queue);
+		{
+			fetchList.push(entry);
+		}
+
+		// Special case: If there are no entries in the fetch list and this is
+		// an update, create an dummy entry, so that we'll get the current count
+		if (fetchList.length === 0 && _isUpdate)
+		{
+			fetchList.push({
+				"start": 0, "count": 0
+			});
+		}
 
 		// Execute all queries
-		for (var i = 0; i < queries.length; i++)
+		for (var i = 0; i < fetchList.length; i++)
 		{
-			// Sanitize the requests
+			// Build the query
-			queries[i].start = Math.max(0, queries[i].start);
+			var query = {
-			queries[i].num_rows = Math.min(this._grid.getTotalCount(),
+					"start": fetchList[i].start,
-					queries[i].start + queries[i].num_rows) - queries[i].start;
+					"num_rows": fetchList[i].count,
+					"refresh": false
+			};
 
 			// Context used in the callback function
 			var ctx = {
 					"self": this,
-					"start": queries[i].start,
+					"start": query.start,
-					"count": queries[i].num_rows
+					"count": query.num_rows,
 			};
 
 			// Call the callback
-			this._dataProvider.dataFetch(queries[i], this._lastModification,
+			this._dataProvider.dataFetch(query, this._fetchCallback, ctx);
-					this._fetchCallback, ctx);
 		}
 
-		// Flush the queue
-		this._queue = [];
 	},
 
 	_clearTimer: function () {
@@ -474,6 +499,7 @@ var et2_dataview_controller = Class.extend({
 					"uid": _order[i],
 					"row": null
 				};
+
 				this._insertDataRow(entry, true);
 
 				// Remember the new entry
@@ -488,16 +514,18 @@ var et2_dataview_controller = Class.extend({
 			}
 		}
 
-		// Delete as many rows as we have left
+		// Delete as many rows as we have left, invalidate the corresponding
+		// index entry
 		for (var i = mapIdx; i < _idxMap.length; i++)
 		{
 			this._grid.deleteRow(idx);
+			_idxMap[i].uid = null;
 		}
 
 		return result;
 	},
 
-	_mergeResult: function (_newEntries, _invalidStartIdx, _diff) {
+	_mergeResult: function (_newEntries, _invalidStartIdx, _diff, _total) {
 
 		if (_newEntries.length > 0 || _diff > 0)
 		{
@@ -510,21 +538,20 @@ var et2_dataview_controller = Class.extend({
 				newMap[_newEntries[i].idx] = _newEntries[i];
 			}
 
-			// Insert all old entries that have a row into the new index map
+			// Merge the old map with all old entries
-			// while adjusting their indices
 			for (var key in this._indexMap)
 			{
 				// Get the corresponding index entry
 				var entry = this._indexMap[key];
 
-				// Only keep index entries which are currently displayed
+				// Calculate the new index -- if rows were deleted, we'll
-				if (entry.row)
+				// have to adjust the index
+				var newIdx = entry.idx >= _invalidStartIdx
+						? entry.idx - _diff : entry.idx;
+				if (newIdx >= 0 && newIdx < _total
+				    && typeof newMap[newIdx] === "undefined")
 				{
-					// Calculate the new index -- if rows were deleted, we'll
+					entry.idx = newIdx;
-					// have to adjust the index
-					var newIdx = entry.idx >= _invalidStartIdx
-							? entry.idx - _diff : entry.idx;
-					entry.idx = key;
 					newMap[newIdx] = entry;
 				}
 			}
@@ -542,9 +569,6 @@ var et2_dataview_controller = Class.extend({
 			return;
 		}
 
-		// Copy the last modification
-		this.self._lastModification = _response.lastModification;
-
 		// Make sure _response.order.length is not longer than the requested
 		// count
 		var order = _response.order.splice(0, this.count);
@@ -560,7 +584,7 @@ var et2_dataview_controller = Class.extend({
 		// Merge the new indices, update all indices with rows that were not
 		// affected and invalidate all indices if there were changes
 		this.self._mergeResult(res, this.start + order.length,
-				idxMap.length - order.length);
+				idxMap.length - order.length, _response.total);
 
 		// Update the total element count in the grid
 		this.self._grid.setTotalCount(_response.total);

etemplate/js/et2_dataview_view_grid.js

@@ -135,6 +135,22 @@ var et2_dataview_grid = et2_dataview_container.extend(et2_dataview_IViewRange, {
 		this.setTotalCount(oldTotalCount);
 	},
 
+	/**
+	 * Throws all elements away which are outside the current view range
+	 */
+	cleanup: function () {
+		// Update the pixel positions
+		this._recalculateElementPosition();
+
+		// Get the visible mapping indices and recalculate index and pixel
+		// position of the containers.
+		var mapVis = this._calculateVisibleMappingIndices();
+
+		// Delete all invisible elements -- if anything changed, we have to
+		// recalculate the pixel positions again
+		this._cleanupOutOfRangeElements(mapVis, 0);
+	},
+
 	/**
 	 * The insertRow function can be called to insert the given container(s) at
 	 * the given row index. If there currently is another container at that
@@ -560,22 +576,24 @@ var et2_dataview_grid = et2_dataview_container.extend(et2_dataview_IViewRange, {
 	/**
 	 * Deletes all elements which are "out of the view range". This function is
 	 * internally used by "_doInvalidate". How many elements that are out of the
-	 * view range get preserved fully depends on the ET2_GRID_HOLD_COUNT
+	 * view range get preserved fully depends on the _holdCount parameter
 	 * variable.
 	 *
 	 * @param _mapVis contains the _map indices of the just visible containers.
+	 * @param _holdCount contains the number of elements that should be kept,
+	 * if not given, this parameter defaults to ET2_GRID_HOLD_COUNT
 	 */
-	_cleanupOutOfRangeElements: function(_mapVis) {
+	_cleanupOutOfRangeElements: function(_mapVis, _holdCount) {
 
 		// Iterates over the map from and to the given indices and pushes all
-		// elements onto the given array, which are more than ET2_GRID_HOLD_COUNT
+		// elements onto the given array, which are more than _holdCount
 		// elements remote from the start.
 		function searchElements(_arr, _start, _stop, _dir)
 		{
 			var dist = 0;
 			for (var i = _start; _dir > 0 ? i <= _stop : i >= _stop; i += _dir)
 			{
-				if (dist > ET2_GRID_HOLD_COUNT)
+				if (dist > _holdCount)
 				{
 					_arr.push(i);
 				}
@@ -586,6 +604,10 @@ var et2_dataview_grid = et2_dataview_container.extend(et2_dataview_IViewRange, {
 			}
 		}
 
+		// Set _holdCount to ET2_GRID_HOLD_COUNT if the parameters is not given
+		_holdCount = typeof _holdCount === "undefined" ? ET2_GRID_HOLD_COUNT :
+				_holdCount;
+
 		// Collect all elements that will be deleted at the top and at the
 		// bottom of the grid
 		var deleteTop = [];

etemplate/js/et2_extension_nextmatch.js

@@ -861,16 +861,17 @@ var et2_nextmatch_header_bar = et2_DOMWidget.extend(et2_INextmatchHeader, {
 			.addClass("header_count ui-corner-all");
 
 		// Need to figure out how to update this as grid scrolls
-		//this.count.append("? - ? ").append(egw.lang("of")).append(" ");
+		// this.count.append("? - ? ").append(egw.lang("of")).append(" ");
 		this.count_total = jQuery(document.createElement("span"))
 			.appendTo(this.count)
 			.text(settings.total + "");
 		this.count.prependTo(this.div);
 
 		// Set up so if row count changes, display is updated
-		nm_div.bind('nm_data', function(e) { // Have to bind to DOM node, not et2 widget
+		// Register the handler which will update the "totalCount" display
-			self.count_total.text(e.nm_data.total);
+		/*this.dataview.grid.setInvalidateCallback(function () {
-		});
+			this.count_total.text(this.dataview.grid.getTotalCount() + "");
+		}, this);*/
 
 		// Left & Right headers
 		this.headers = [];

etemplate/js/et2_extension_nextmatch_controller.js

@@ -108,8 +108,7 @@ var et2_nextmatch_controller = et2_dataview_controller.extend(
 	/** -- Implementation of et2_IDataProvider -- **/
 
 
-	dataFetch: function (_queriedRange, _lastModification, _callback,
+	dataFetch: function (_queriedRange, _callback, _context) {
-			_context) {
 		// Pass the fetch call to the API, multiplex the data about the
 		// nextmatch instance into the call.
 		this.egw.dataFetch(
@@ -117,7 +116,6 @@ var et2_nextmatch_controller = et2_dataview_controller.extend(
 				_queriedRange,
 				this._filters,
 				this._widgetId,
-				_lastModification,
 				_callback,
 				_context);
 	},

etemplate/js/et2_widget_image.js

@@ -83,7 +83,7 @@ var et2_image = et2_baseWidget.extend([et2_IDetachedDOM], {
 		if(this.options.href)
 		{
 			this.egw().call_link(this.options.href, this.options.extra_link_target, this.options.extra_link_popup);
-		}		
+		}
 	},
 
 	transformAttributes: function(_attrs) {
@@ -124,7 +124,7 @@ var et2_image = et2_baseWidget.extend([et2_IDetachedDOM], {
 		this.options.src = _value;
 
 		var src = this.egw().image(_value);
-		if(src)
+		if (src)
 		{
 			this.image.attr("src", src).show();
 			return true;

etemplate/js/test/test_dataview.html

@@ -95,11 +95,10 @@
 
 			var dataprovider = Class.extend(et2_IDataProvider, {
 
-				dataFetch: function (_queriedRange, _lastModification, _callback, _context) {
+				dataFetch: function (_queriedRange, _callback, _context) {
 					var response = {
 						"order": data.slice(_queriedRange.start, _queriedRange.start + _queriedRange.num_rows),
-						"total": data.length,
+						"total": data.length
-						"lastModification": 0
 					};
 
 					window.setTimeout(function () {

phpgwapi/js/jsapi/egw_data.js

@@ -76,7 +76,6 @@ egw.extend("data", egw.MODULE_APP_LOCAL, function (_app, _wnd) {
 			{
 				_callback.call(_context, {
 					"order": _result.order,
-					"lastModification": _result.lastModification,
 					"total": parseInt(_result.total),
 					"readonlys": _result.readonlys,
 					"rows": _result.rows
@@ -126,10 +125,6 @@ egw.extend("data", egw.MODULE_APP_LOCAL, function (_app, _wnd) {
 		 * @param knownUids is an array of uids already known to the client. 
 		 *  This parameter may be null in order to indicate that the client 
 		 *  currently has no data for the given filter settings.
-		 * @param lastModification is the last timestamp that was returned from
-		 * 	the server and for which the client has data. It may be null in
-		 * 	order to indicate, that the client currently has no data or needs a
-		 * 	complete refresh.
 		 * @param callback is the function that should get called, once the data
 		 * 	is available. The data passed to the callback function has the
 		 * 	following form:
@@ -148,7 +143,7 @@ egw.extend("data", egw.MODULE_APP_LOCAL, function (_app, _wnd) {
 		 * 	called.
 		 */
 		dataFetch: function (_execId, _queriedRange, _filters, _widgetId,
-				_lastModification, _callback, _context)
+				_callback, _context)
 		{
 			var request = egw.json(
 				"etemplate_widget_nextmatch::ajax_get_rows::etemplate",