From 0abec539c32aad2cb1cc27f8550f04319a70a632 Mon Sep 17 00:00:00 2001
From: Ralf Becker <RalfBecker@outdoor-training.de>
Date: Fri, 1 Mar 2019 15:46:54 +0100
Subject: [PATCH] * All apps: store history of multiline fields as diff only
 (migration can take 15min!)

---
 api/setup/setup.inc.php         |  2 +-
 api/setup/tables_update.inc.php | 54 +++++++++++++++++++++++++++++++++
 api/src/Storage/History.php     |  6 ++--
 3 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/api/setup/setup.inc.php b/api/setup/setup.inc.php
index 20ce891239..b547569510 100644
--- a/api/setup/setup.inc.php
+++ b/api/setup/setup.inc.php
@@ -11,7 +11,7 @@
 /* Basic information about this app */
 $setup_info['api']['name']      = 'api';
 $setup_info['api']['title']     = 'EGroupware API';
-$setup_info['api']['version']   = '17.1.005';
+$setup_info['api']['version']   = '17.1.006';
 $setup_info['api']['versions']['current_header'] = '1.29';
 // maintenance release in sync with changelog in doc/rpm-build/debian.changes
 $setup_info['api']['versions']['maintenance_release'] = '17.1.20190222';
diff --git a/api/setup/tables_update.inc.php b/api/setup/tables_update.inc.php
index c78a0ed59d..bcdc14b049 100644
--- a/api/setup/tables_update.inc.php
+++ b/api/setup/tables_update.inc.php
@@ -516,3 +516,57 @@ function api_upgrade17_1_004()
 	return $GLOBALS['setup_info']['api']['currentver'] = '17.1.005';
 }
 
+
+/**
+ * Store multiline history content as diff
+ *
+ * Benchmark on Ralf's Laptop (OS X)
+ * - auto/Native: 66k rows in 15mins
+ *
+ * @return string new version
+ */
+function api_upgrade17_1_005()
+{
+	$renderer = new Horde_Text_Diff_Renderer_Unified();
+	$start = microtime(true);
+	$junk_size = 200;	// 2*200*160KB = 64MB
+	$total = $saved = 0;
+	do {
+		$n = 0;
+		foreach($GLOBALS['egw_setup']->db->select('egw_history_log', 'history_id,history_new_value,history_old_value', array(
+			'history_old_value != '.$GLOBALS['egw_setup']->db->quote(Api\Storage\Tracking::DIFF_MARKER),
+			// if one is empty, no need to store diff
+			"(LENGTH(history_new_value) > 0 AND LENGTH(history_old_value) > 0)",
+			"(history_status LIKE '%description' OR history_status='De' OR history_status='note'".
+				" OR LENGTH(history_new_value) > 200 OR LENGTH(history_old_value) > 200)",
+		), __LINE__, __FILE__, 0, 'ORDER BY history_id', false, $junk_size) as $row)
+		{
+			// use OS diff command for big texts, if available
+			$diff = new Horde_Text_Diff('auto', array(
+				explode("\n", $row['history_old_value']),
+				explode("\n", $row['history_new_value']),
+			));
+			$diff_str = $renderer->render($diff);
+
+			$saved += strlen($row['history_old_value'])+strlen($row['history_new_value'])
+				-strlen($diff_str)-strlen(Api\Storage\Tracking::DIFF_MARKER);
+
+			$GLOBALS['egw_setup']->db->update('egw_history_log', array(
+				'history_new_value' => $diff_str,
+				'history_old_value' => Api\Storage\Tracking::DIFF_MARKER,
+			), array(
+				'history_id' => $row['history_id'],
+			), __LINE__, __FILE__);
+
+			$n++;
+			$total++;
+		}
+	}
+	while($n == $junk_size);
+	$saved = number_format($saved/(1024.0*1024.0), 1);
+	$time = number_format((microtime(true)-$start)/60, 1);
+	echo "$total history-records converted in $time minutes to diff with a total of $saved MB saved\n";
+
+	return $GLOBALS['setup_info']['api']['currentver'] = '17.1.006';
+}
+
diff --git a/api/src/Storage/History.php b/api/src/Storage/History.php
index b50f3ae4cf..9d29df9f60 100644
--- a/api/src/Storage/History.php
+++ b/api/src/Storage/History.php
@@ -311,7 +311,9 @@ class History
 
 	protected static function needs_diff($name, $value)
 	{
-		return $name == 'note' || $name == 'description' ||
-			($value && (strlen($value) > 50 || strstr($value, "\n") !== FALSE));
+		return $name == 'note' ||	// Addressbook
+			strpos($name, 'description') !== false ||	// Calendar, Records, Timesheet, ProjectManager, Resources
+			$name == 'De' ||	// Tracker, InfoLog
+			($value && (strlen($value) > 200 || strstr($value, "\n") !== FALSE));
 	}
 }