transform: add truncate_keep_extension and truncate_bytes

This change adds a truncate_bytes mode which counts the number of bytes, as
opposed to the number of UTF-8 characters. This can be useful for ensuring that a
crypt-encoded filename will not exceed the underlying backend's length limits
(see https://forum.rclone.org/t/any-clear-file-name-length-when-using-crypt/36930 ).

This change also adds support for _keep_extension when using truncate and
truncate_bytes.

lib/transform/transform.go

@@ -165,14 +165,25 @@ func transformPathSegment(s string, t transform) (string, error) {
 		if err != nil {
 			return s, err
 		}
-		if max <= 0 {
-			return s, nil
+		return truncateChars(s, max, false), nil
+	case ConvTruncateKeepExtension:
+		max, err := strconv.Atoi(t.value)
+		if err != nil {
+			return s, err
 		}
-		if utf8.RuneCountInString(s) <= max {
-			return s, nil
+		return truncateChars(s, max, true), nil
+	case ConvTruncateBytes:
+		max, err := strconv.Atoi(t.value)
+		if err != nil {
+			return s, err
 		}
-		runes := []rune(s)
-		return string(runes[:max]), nil
+		return truncateBytes(s, max, false)
+	case ConvTruncateBytesKeepExtension:
+		max, err := strconv.Atoi(t.value)
+		if err != nil {
+			return s, err
+		}
+		return truncateBytes(s, max, true)
 	case ConvEncoder:
 		var enc encoder.MultiEncoder
 		err := enc.Set(t.value)
@@ -231,9 +242,13 @@ func transformPathSegment(s string, t transform) (string, error) {
 //
 // i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
 func SuffixKeepExtension(remote string, suffix string) string {
+	base, exts := splitExtension(remote)
+	return base + suffix + exts
+}
+
+func splitExtension(remote string) (base, exts string) {
+	base = remote
 	var (
-		base  = remote
-		exts  = ""
 		first = true
 		ext   = path.Ext(remote)
 	)
@@ -248,7 +263,45 @@ func SuffixKeepExtension(remote string, suffix string) string {
 		first = false
 		ext = path.Ext(base)
 	}
-	return base + suffix + exts
+	return base, exts
+}
+
+func truncateChars(s string, max int, keepExtension bool) string {
+	if max <= 0 {
+		return s
+	}
+	if utf8.RuneCountInString(s) <= max {
+		return s
+	}
+	exts := ""
+	if keepExtension {
+		s, exts = splitExtension(s)
+	}
+	runes := []rune(s)
+	return string(runes[:max-utf8.RuneCountInString(exts)]) + exts
+}
+
+// truncateBytes is like truncateChars but counts the number of bytes, not UTF-8 characters
+func truncateBytes(s string, max int, keepExtension bool) (string, error) {
+	if max <= 0 {
+		return s, nil
+	}
+	if len(s) <= max {
+		return s, nil
+	}
+	exts := ""
+	if keepExtension {
+		s, exts = splitExtension(s)
+	}
+
+	// ensure we don't split a multi-byte UTF-8 character
+	for i := max - len(exts); i > 0; i-- {
+		b := append([]byte(s)[:i], exts...)
+		if len(b) <= max && utf8.Valid(b) {
+			return string(b), nil
+		}
+	}
+	return "", errors.New("could not truncate to valid UTF-8")
 }
 
 // forbid transformations that add/remove path separators