From d068e0b1a9e587ddc669e0a795d1dcee99defd74 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 12 Jun 2024 10:52:58 +0100 Subject: [PATCH] operations: fix hashing problem in integration tests Before this change backends which supported more than one hash (eg pcloud) or backends which wrapped backends supporting more than one hash (combine) would fail the TestMultithreadCopy and TestMultithreadCopyAbort with an error like Failed to make new multi hasher: requested set 000001ff contains unknown hash types This was caused by the tests limiting the globally available hashes to the first hash supplied by the backend. This was added in this commit d5d28a7513b09701 operations: fix overwrite of destination when multi-thread transfer fails to overcome the tests taking >100s on the local backend because they made every single hash that the local backend. It brought this time down to 20s. This commit fixes the problem and retains the CPU speedup by only applying the fix from the original commit if the destination backend is the local backend. This fixes the common case (testing on the local backend). This does not fix the problem for a backend which wraps the local backend (eg combine) but this is run only on the integration test machine and not on all the CI. --- fs/operations/multithread_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go index 6b4c86f7d..3dc2a9ba8 100644 --- a/fs/operations/multithread_test.go +++ b/fs/operations/multithread_test.go @@ -121,11 +121,13 @@ func skipIfNotMultithread(ctx context.Context, t *testing.T, r *fstest.Run) int t.Skip("multithread writing not supported") } - // Only support one hash otherwise we end up spending a huge amount of CPU on hashing! - oldHashes := hash.SupportOnly([]hash.Type{r.Fremote.Hashes().GetOne()}) - t.Cleanup(func() { - _ = hash.SupportOnly(oldHashes) - }) + // Only support one hash for the local backend otherwise we end up spending a huge amount of CPU on hashing! + if r.Fremote.Features().IsLocal { + oldHashes := hash.SupportOnly([]hash.Type{r.Fremote.Hashes().GetOne()}) + t.Cleanup(func() { + _ = hash.SupportOnly(oldHashes) + }) + } ci := fs.GetConfig(ctx) chunkSize := int(ci.MultiThreadChunkSize)