From 83e2d7022216bffcbd31d6088e204202ce9c0413 Mon Sep 17 00:00:00 2001
From: nathangray <nathangray.bsc+github@gmail.com>
Date: Fri, 22 Nov 2019 09:44:27 -0700
Subject: [PATCH] Mail: Fix bullet & number to text conversion failed with
 non-ascii characters

---
 api/src/Mail/Html.php       | 19 ++++++++++++++++---
 api/tests/Mail/HtmlTest.php | 32 +++++++++++++++++++++++++++++---
 2 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/api/src/Mail/Html.php b/api/src/Mail/Html.php
index 0ec923a5e0..ceb7ff0c34 100644
--- a/api/src/Mail/Html.php
+++ b/api/src/Mail/Html.php
@@ -492,7 +492,11 @@ class Html
 			return $html;
 		}
 
-		$dom = \DOMDocument::loadHTML($html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD | LIBXML_NOBLANKS);
+		$dom = new \DOMDocument('1.0','UTF-8');
+		$dom->loadHTML(
+				'<?xml encoding="UTF-8">'. Api\Translation::convert($html,false, 'utf8'),
+				LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD | LIBXML_NOBLANKS
+		);
 		if(!$dom)
 		{
 			// Failed to parse
@@ -508,14 +512,23 @@ class Html
 				$list_text = "\r\n";
 				$item_count = 0;
 				$prefix = $list_type == 'ul' ? ' * ' : '. ';
-				foreach($list->getElementsByTagName('li') as $element)
+				$elements = $list->getElementsByTagName('li');
+				foreach($elements as $element)
 				{
-					$list_text .= ($list_type == 'ol' ? ' '. ++$item_count : '') . $prefix . $element->textContent . "\r\n";
+					if($element->tagName == 'li')
+					{
+						$list_text .= ($list_type == 'ol' ? ' '. ++$item_count : '') . $prefix . $element->textContent . "\r\n";
+					}
+					else
+					{
+						$list_text .= $element->nodeValue;
+					}
 				}
 
 				$list->parentNode->replaceChild($dom->createTextNode($list_text), $list);
 			}
 		}
+		$dom->removeChild($dom->firstChild);
 
 		return $dom->saveHTML();
 	}
diff --git a/api/tests/Mail/HtmlTest.php b/api/tests/Mail/HtmlTest.php
index 7e791d4b23..4808fc796c 100644
--- a/api/tests/Mail/HtmlTest.php
+++ b/api/tests/Mail/HtmlTest.php
@@ -50,17 +50,43 @@ class HtmlTest extends TestCase {
 			['Not actually HTML', 'Not actually HTML'],
 			['HTML, but <b>NO</b> list here', 'HTML, but <b>NO</b> list here'],
 			["<p>Unordered list:<ul><li>First</li>\r\n<li>Second</li>\r\n<li>Third</li>\r\n</ul>\r\nPost text</p>",
-				"<p>Unordered list:\r\n * First\r\n * Second\r\n * Third\r\n<p>\r\nPost text</p></p>\n"],
+				"<p>Unordered list:</p>\r\n * First\r\n * Second\r\n * Third\r\n<p>\r\nPost text</p>\n"],
 			["Ordered list:".
 				"<ol><li>First</li>\r\n"
 				. "<li>Second</li>\r\n"
 				. "<li>Third</li>\r\n"
 				. "</ol>Post text",
-				"<p>Ordered list:\r\n"
+				"<p>Ordered list:</p>\r\n"
 				. " 1. First\r\n"
 				. " 2. Second\r\n"
 				. " 3. Third\r\n"
-				. "<p>Post text</p></p>\n"],
+				. "<p>Post text</p>\n"],
+			[
+				'
+				<ol>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">Nummerierung&nbsp;</span></li>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">zweitens</span></li>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">drittens</span></li>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">viertens</span></li>
+				</ol>
+				<p><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">dann normaler Text</span></p>
+				<ul>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">aufz&auml;hlung</span></li>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">mit verschiedenen punkten</span></li>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">dritter spiegelstrich</span></li>
+				<li><span style="width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;">vierter strich</span></li>
+				</ul>
+				<pre>&nbsp;</pre>',
+				"\r\n 1. Nummerierung&nbsp;\r
+ 2. zweitens\r
+ 3. drittens\r
+ 4. viertens\r
+<p><span style=\"width: 100%; display: inline; font-family: arial, helvetica, sans-serif; font-size: 12pt;\">dann normaler Text</span></p>\r
+ * aufz&auml;hlung\r
+ * mit verschiedenen punkten\r
+ * dritter spiegelstrich\r
+ * vierter strich\r\n<pre>&nbsp;</pre>\n"
+			]
 		);
 	}
 }