rclone test info: add --check-base32768 flag to check can store all base32768 characters

Fixes #7208

cmd/test/info/base32768.go

@@ -0,0 +1,94 @@
+package info
+
+// Create files with all possible base 32768 file names
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fspath"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
+)
+
+const safeAlphabet = "ƀɀɠʀҠԀڀڠݠހ߀ကႠᄀᄠᅀᆀᇠሀሠበዠጠᎠᏀᐠᑀᑠᒀᒠᓀᓠᔀᔠᕀᕠᖀᖠᗀᗠᘀᘠᙀᚠᛀកᠠᡀᣀᦀ᧠ᨠᯀᰀᴀ⇠⋀⍀⍠⎀⎠⏀␀─┠╀╠▀■◀◠☀☠♀♠⚀⚠⛀⛠✀✠❀➀➠⠀⠠⡀⡠⢀⢠⣀⣠⤀⤠⥀⥠⦠⨠⩀⪀⪠⫠⬀⬠⭀ⰀⲀⲠⳀⴀⵀ⺠⻀㇀㐀㐠㑀㑠㒀㒠㓀㓠㔀㔠㕀㕠㖀㖠㗀㗠㘀㘠㙀㙠㚀㚠㛀㛠㜀㜠㝀㝠㞀㞠㟀㟠㠀㠠㡀㡠㢀㢠㣀㣠㤀㤠㥀㥠㦀㦠㧀㧠㨀㨠㩀㩠㪀㪠㫀㫠㬀㬠㭀㭠㮀㮠㯀㯠㰀㰠㱀㱠㲀㲠㳀㳠㴀㴠㵀㵠㶀㶠㷀㷠㸀㸠㹀㹠㺀㺠㻀㻠㼀㼠㽀㽠㾀㾠㿀㿠䀀䀠䁀䁠䂀䂠䃀䃠䄀䄠䅀䅠䆀䆠䇀䇠䈀䈠䉀䉠䊀䊠䋀䋠䌀䌠䍀䍠䎀䎠䏀䏠䐀䐠䑀䑠䒀䒠䓀䓠䔀䔠䕀䕠䖀䖠䗀䗠䘀䘠䙀䙠䚀䚠䛀䛠䜀䜠䝀䝠䞀䞠䟀䟠䠀䠠䡀䡠䢀䢠䣀䣠䤀䤠䥀䥠䦀䦠䧀䧠䨀䨠䩀䩠䪀䪠䫀䫠䬀䬠䭀䭠䮀䮠䯀䯠䰀䰠䱀䱠䲀䲠䳀䳠䴀䴠䵀䵠䶀䷀䷠一丠乀习亀亠什仠伀传佀你侀侠俀俠倀倠偀偠傀傠僀僠儀儠兀兠冀冠净几刀删剀剠劀加勀勠匀匠區占厀厠叀叠吀吠呀呠咀咠哀哠唀唠啀啠喀喠嗀嗠嘀嘠噀噠嚀嚠囀因圀圠址坠垀垠埀埠堀堠塀塠墀墠壀壠夀夠奀奠妀妠姀姠娀娠婀婠媀媠嫀嫠嬀嬠孀孠宀宠寀寠尀尠局屠岀岠峀峠崀崠嵀嵠嶀嶠巀巠帀帠幀幠庀庠廀廠开张彀彠往徠忀忠怀怠恀恠悀悠惀惠愀愠慀慠憀憠懀懠戀戠所扠技抠拀拠挀挠捀捠掀掠揀揠搀搠摀摠撀撠擀擠攀攠敀敠斀斠旀无昀映晀晠暀暠曀曠最朠杀杠枀枠柀柠栀栠桀桠梀梠检棠椀椠楀楠榀榠槀槠樀樠橀橠檀檠櫀櫠欀欠歀歠殀殠毀毠氀氠汀池沀沠泀泠洀洠浀浠涀涠淀淠渀渠湀湠満溠滀滠漀漠潀潠澀澠激濠瀀瀠灀灠炀炠烀烠焀焠煀煠熀熠燀燠爀爠牀牠犀犠狀狠猀猠獀獠玀玠珀珠琀琠瑀瑠璀璠瓀瓠甀甠畀畠疀疠痀痠瘀瘠癀癠皀皠盀盠眀眠着睠瞀瞠矀矠砀砠础硠碀碠磀磠礀礠祀祠禀禠秀秠稀稠穀穠窀窠竀章笀笠筀筠简箠節篠簀簠籀籠粀粠糀糠紀素絀絠綀綠緀締縀縠繀繠纀纠绀绠缀缠罀罠羀羠翀翠耀耠聀聠肀肠胀胠脀脠腀腠膀膠臀臠舀舠艀艠芀芠苀苠茀茠荀荠莀莠菀菠萀萠葀葠蒀蒠蓀蓠蔀蔠蕀蕠薀薠藀藠蘀蘠虀虠蚀蚠蛀蛠蜀蜠蝀蝠螀螠蟀蟠蠀蠠血衠袀袠裀裠褀褠襀襠覀覠觀觠言訠詀詠誀誠諀諠謀謠譀譠讀讠诀诠谀谠豀豠貀負賀賠贀贠赀赠趀趠跀跠踀踠蹀蹠躀躠軀軠輀輠轀轠辀辠迀迠退造遀遠邀邠郀郠鄀鄠酀酠醀醠釀釠鈀鈠鉀鉠銀銠鋀鋠錀錠鍀鍠鎀鎠鏀鏠鐀鐠鑀鑠钀钠铀铠销锠镀镠門閠闀闠阀阠陀陠隀隠雀雠需霠靀靠鞀鞠韀韠頀頠顀顠颀颠飀飠餀餠饀饠馀馠駀駠騀騠驀驠骀骠髀髠鬀鬠魀魠鮀鮠鯀鯠鰀鰠鱀鱠鲀鲠鳀鳠鴀鴠鵀鵠鶀鶠鷀鷠鸀鸠鹀鹠麀麠黀黠鼀鼠齀齠龀龠ꀀꀠꁀꁠꂀꂠꃀꃠꄀꄠꅀꅠꆀꆠꇀꇠꈀꈠꉀꉠꊀꊠꋀꋠꌀꌠꍀꍠꎀꎠꏀꏠꐀꐠꑀꑠ꒠ꔀꔠꕀꕠꖀꖠꗀꗠꙀꚠꛀ꜀꜠ꝀꞀꡀ"
+
+func (r *results) checkBase32768() {
+	r.canBase32768 = false
+	ctx := context.Background()
+
+	n := 0
+	dir, err := os.MkdirTemp("", "rclone-base32768-files")
+	if err != nil {
+		log.Printf("Failed to make temp dir: %v", err)
+		return
+	}
+	defer func() {
+		_ = os.RemoveAll(dir)
+	}()
+
+	// Create test files
+	for _, c := range safeAlphabet {
+		var out strings.Builder
+		for i := rune(0); i < 32; i++ {
+			out.WriteRune(c + i)
+		}
+		fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
+		err = os.WriteFile(fileName, []byte(fileName), 0666)
+		if err != nil {
+			log.Printf("write %q failed: %v", fileName, err)
+			return
+		}
+		n++
+	}
+
+	// Make a local fs
+	fLocal, err := fs.NewFs(ctx, dir)
+	if err != nil {
+		log.Printf("Failed to make local fs: %v", err)
+		return
+	}
+
+	testDir := "test-base32768"
+
+	// Make a remote fs
+	s := fs.ConfigStringFull(r.f)
+	s = fspath.JoinRootPath(s, testDir)
+	fRemote, err := fs.NewFs(ctx, s)
+	if err != nil {
+		log.Printf("Failed to make remote fs: %v", err)
+		return
+	}
+
+	defer func() {
+		err := operations.Purge(ctx, r.f, testDir)
+		if err != nil {
+			log.Printf("Failed to purge test directory: %v", err)
+			return
+		}
+	}()
+
+	// Sync local to remote
+	err = sync.Sync(ctx, fRemote, fLocal, false)
+	if err != nil {
+		log.Printf("Failed to sync remote fs: %v", err)
+		return
+	}
+
+	// Check local to remote
+	err = operations.Check(ctx, &operations.CheckOpt{
+		Fdst: fRemote,
+		Fsrc: fLocal,
+	})
+	if err != nil {
+		log.Printf("Failed to check remote fs: %v", err)
+		return
+	}
+
+	r.canBase32768 = true
+}

cmd/test/info/info.go

@@ -37,6 +37,7 @@ var (
 	checkControl       bool
 	checkLength        bool
 	checkStreaming     bool
+	checkBase32768     bool
 	all                bool
 	uploadWait         time.Duration
 	positionLeftRe     = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
@@ -53,6 +54,7 @@ func init() {
 	flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file", "")
 	flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length", "")
 	flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size", "")
+	flags.BoolVarP(cmdFlags, &checkBase32768, "check-base32768", "", false, "Check can store all possible base32768 characters", "")
 	flags.BoolVarP(cmdFlags, &all, "all", "", false, "Run all tests", "")
 }
 
@@ -71,7 +73,7 @@ a bit of go code for each one.
 	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1e6, command, args)
-		if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !all {
+		if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !checkBase32768 && !all {
 			log.Fatalf("no tests selected - select a test or use --all")
 		}
 		if all {
@@ -79,6 +81,7 @@ a bit of go code for each one.
 			checkControl = true
 			checkLength = true
 			checkStreaming = true
+			checkBase32768 = true
 		}
 		for i := range args {
 			f := cmd.NewFsDir(args[i : i+1])
@@ -100,6 +103,7 @@ type results struct {
 	canReadUnnormalized  bool
 	canReadRenormalized  bool
 	canStream            bool
+	canBase32768         bool
 }
 
 func newResults(ctx context.Context, f fs.Fs) *results {
@@ -141,6 +145,9 @@ func (r *results) Print() {
 	if checkStreaming {
 		fmt.Printf("canStream = %v\n", r.canStream)
 	}
+	if checkBase32768 {
+		fmt.Printf("base32768isOK = %v // make sure maxFileLength for 2 byte unicode chars is the same as for 1 byte characters\n", r.canBase32768)
+	}
 }
 
 // WriteJSON writes the results to a JSON file when requested
@@ -483,6 +490,9 @@ func readInfo(ctx context.Context, f fs.Fs) error {
 	if checkStreaming {
 		r.checkStreaming()
 	}
+	if checkBase32768 {
+		r.checkBase32768()
+	}
 	r.Print()
 	r.WriteJSON()
 	return nil