package onedrive import ( "context" "encoding/json" "fmt" "slices" "testing" "time" _ "github.com/rclone/rclone/backend/local" "github.com/rclone/rclone/backend/onedrive/api" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/fstest/fstests" "github.com/rclone/rclone/lib/random" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // go test -timeout 30m -run ^TestIntegration/FsMkdir/FsPutFiles/Internal$ github.com/rclone/rclone/backend/onedrive -remote TestOneDrive:meta -v // go test -timeout 30m -run ^TestIntegration/FsMkdir/FsPutFiles/Internal$ github.com/rclone/rclone/backend/onedrive -remote TestOneDriveBusiness:meta -v // go run ./fstest/test_all -remotes TestOneDriveBusiness:meta,TestOneDrive:meta -verbose -maxtries 1 var ( t1 = fstest.Time("2023-08-26T23:13:06.499999999Z") t2 = fstest.Time("2020-02-29T12:34:56.789Z") t3 = time.Date(1994, time.December, 24, 9+12, 0, 0, 525600, time.FixedZone("Eastern Standard Time", -5)) ctx = context.Background() content = "hello" ) const ( testUserID = "ryan@contoso.com" // demo user from doc examples (can't share files with yourself) // https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1 ) // TestMain drives the tests func TestMain(m *testing.M) { fstest.TestMain(m) } // TestWritePermissions tests reading and writing permissions func (f *Fs) TestWritePermissions(t *testing.T, r *fstest.Run) { // setup ctx, ci := fs.AddConfig(ctx) ci.Metadata = true _ = f.opt.MetadataPermissions.Set("read,write") file1 := r.WriteFile(randomFilename(), content, t2) // add a permission with "read" role permissions := defaultPermissions(f.driveType) permissions[0].Roles[0] = api.ReadRole expectedMeta, actualMeta := f.putWithMeta(ctx, t, &file1, permissions) f.compareMeta(t, expectedMeta, actualMeta, false) expectedP, actualP := unmarshalPerms(t, expectedMeta["permissions"]), unmarshalPerms(t, actualMeta["permissions"]) found, num := false, 0 foundCount := 0 for i, p := range actualP { for _, identity := range p.GetGrantedToIdentities(f.driveType) { if identity.User.DisplayName == testUserID { // note: expected will always be element 0 here, but actual may be variable based on org settings assert.Equal(t, expectedP[0].Roles, p.Roles) found, num = true, i foundCount++ } } if f.driveType == driveTypePersonal { if p.GetGrantedTo(f.driveType) != nil && p.GetGrantedTo(f.driveType).User != (api.Identity{}) && p.GetGrantedTo(f.driveType).User.ID == testUserID { // shows up in a different place on biz vs. personal assert.Equal(t, expectedP[0].Roles, p.Roles) found, num = true, i foundCount++ } } } assert.True(t, found, fmt.Sprintf("no permission found with expected role (want: \n\n%v \n\ngot: \n\n%v\n\n)", indent(t, expectedMeta["permissions"]), indent(t, actualMeta["permissions"]))) assert.Equal(t, 1, foundCount, "expected to find exactly 1 match") // update it to "write" permissions = actualP permissions[num].Roles[0] = api.WriteRole expectedMeta, actualMeta = f.putWithMeta(ctx, t, &file1, permissions) f.compareMeta(t, expectedMeta, actualMeta, false) if f.driveType != driveTypePersonal { // zero out some things we expect to be different expectedP, actualP = unmarshalPerms(t, expectedMeta["permissions"]), unmarshalPerms(t, actualMeta["permissions"]) normalize(expectedP) normalize(actualP) expectedMeta.Set("permissions", marshalPerms(t, expectedP)) actualMeta.Set("permissions", marshalPerms(t, actualP)) } assert.JSONEq(t, expectedMeta["permissions"], actualMeta["permissions"]) // remove it permissions[num] = nil _, actualMeta = f.putWithMeta(ctx, t, &file1, permissions) if f.driveType == driveTypePersonal { perms, ok := actualMeta["permissions"] assert.False(t, ok, fmt.Sprintf("permissions metadata key was unexpectedly found: %v", perms)) return } _, actualP = unmarshalPerms(t, expectedMeta["permissions"]), unmarshalPerms(t, actualMeta["permissions"]) found = false var foundP *api.PermissionsType for _, p := range actualP { if p.GetGrantedTo(f.driveType) == nil || p.GetGrantedTo(f.driveType).User == (api.Identity{}) || p.GetGrantedTo(f.driveType).User.ID != testUserID { continue } found = true foundP = p } assert.False(t, found, fmt.Sprintf("permission was found but expected to be removed: %v", foundP)) } // TestUploadSinglePart tests reading/writing permissions using uploadSinglepart() // This is only used when file size is exactly 0. func (f *Fs) TestUploadSinglePart(t *testing.T, r *fstest.Run) { content = "" f.TestWritePermissions(t, r) content = "hello" } // TestReadPermissions tests that no permissions are written when --onedrive-metadata-permissions has "read" but not "write" func (f *Fs) TestReadPermissions(t *testing.T, r *fstest.Run) { // setup ctx, ci := fs.AddConfig(ctx) ci.Metadata = true file1 := r.WriteFile(randomFilename(), "hello", t2) // try adding a permission without --onedrive-metadata-permissions -- should fail // test that what we got before vs. after is the same _ = f.opt.MetadataPermissions.Set("read") _, expectedMeta := f.putWithMeta(ctx, t, &file1, []*api.PermissionsType{}) // return var intentionally switched here permissions := defaultPermissions(f.driveType) _, actualMeta := f.putWithMeta(ctx, t, &file1, permissions) if f.driveType == driveTypePersonal { perms, ok := actualMeta["permissions"] assert.False(t, ok, fmt.Sprintf("permissions metadata key was unexpectedly found: %v", perms)) return } assert.JSONEq(t, expectedMeta["permissions"], actualMeta["permissions"]) } // TestReadMetadata tests that all the read-only system properties are present and non-blank func (f *Fs) TestReadMetadata(t *testing.T, r *fstest.Run) { // setup ctx, ci := fs.AddConfig(ctx) ci.Metadata = true file1 := r.WriteFile(randomFilename(), "hello", t2) permissions := defaultPermissions(f.driveType) _ = f.opt.MetadataPermissions.Set("read,write") _, actualMeta := f.putWithMeta(ctx, t, &file1, permissions) optionals := []string{"package-type", "shared-by-id", "shared-scope", "shared-time", "shared-owner-id"} // not always present for k := range systemMetadataInfo { if slices.Contains(optionals, k) { continue } if k == "description" && f.driveType != driveTypePersonal { continue // not supported } gotV, ok := actualMeta[k] assert.True(t, ok, fmt.Sprintf("property is missing: %v", k)) assert.NotEmpty(t, gotV, fmt.Sprintf("property is blank: %v", k)) } } // TestDirectoryMetadata tests reading and writing modtime and other metadata and permissions for directories func (f *Fs) TestDirectoryMetadata(t *testing.T, r *fstest.Run) { // setup ctx, ci := fs.AddConfig(ctx) ci.Metadata = true _ = f.opt.MetadataPermissions.Set("read,write") permissions := defaultPermissions(f.driveType) permissions[0].Roles[0] = api.ReadRole expectedMeta := fs.Metadata{ "mtime": t1.Format(timeFormatOut), "btime": t2.Format(timeFormatOut), "content-type": dirMimeType, "description": "that is so meta!", } b, err := json.MarshalIndent(permissions, "", "\t") assert.NoError(t, err) expectedMeta.Set("permissions", string(b)) compareDirMeta := func(expectedMeta, actualMeta fs.Metadata, ignoreID bool) { f.compareMeta(t, expectedMeta, actualMeta, ignoreID) // check that all required system properties are present optionals := []string{"package-type", "shared-by-id", "shared-scope", "shared-time", "shared-owner-id"} // not always present for k := range systemMetadataInfo { if slices.Contains(optionals, k) { continue } if k == "description" && f.driveType != driveTypePersonal { continue // not supported } gotV, ok := actualMeta[k] assert.True(t, ok, fmt.Sprintf("property is missing: %v", k)) assert.NotEmpty(t, gotV, fmt.Sprintf("property is blank: %v", k)) } } newDst, err := operations.MkdirMetadata(ctx, f, "subdir", expectedMeta) assert.NoError(t, err) require.NotNil(t, newDst) assert.Equal(t, "subdir", newDst.Remote()) actualMeta, err := fs.GetMetadata(ctx, newDst) assert.NoError(t, err) assert.NotNil(t, actualMeta) compareDirMeta(expectedMeta, actualMeta, false) // modtime fstest.AssertTimeEqualWithPrecision(t, newDst.Remote(), t1, newDst.ModTime(ctx), f.Precision()) // try changing it and re-check it newDst, err = operations.SetDirModTime(ctx, f, newDst, "", t2) assert.NoError(t, err) fstest.AssertTimeEqualWithPrecision(t, newDst.Remote(), t2, newDst.ModTime(ctx), f.Precision()) // ensure that f.DirSetModTime also works err = f.DirSetModTime(ctx, "subdir", t3) assert.NoError(t, err) entries, err := f.List(ctx, "") assert.NoError(t, err) entries.ForDir(func(dir fs.Directory) { if dir.Remote() == "subdir" { fstest.AssertTimeEqualWithPrecision(t, dir.Remote(), t3, dir.ModTime(ctx), f.Precision()) } }) // test updating metadata on existing dir actualMeta, err = fs.GetMetadata(ctx, newDst) // get fresh info as we've been changing modtimes assert.NoError(t, err) expectedMeta = actualMeta expectedMeta.Set("description", "metadata is fun!") expectedMeta.Set("btime", t3.Format(timeFormatOut)) expectedMeta.Set("mtime", t1.Format(timeFormatOut)) expectedMeta.Set("content-type", dirMimeType) perms := unmarshalPerms(t, expectedMeta["permissions"]) perms[0].Roles[0] = api.WriteRole b, err = json.MarshalIndent(perms, "", "\t") assert.NoError(t, err) expectedMeta.Set("permissions", string(b)) newDst, err = operations.MkdirMetadata(ctx, f, "subdir", expectedMeta) assert.NoError(t, err) require.NotNil(t, newDst) assert.Equal(t, "subdir", newDst.Remote()) actualMeta, err = fs.GetMetadata(ctx, newDst) assert.NoError(t, err) assert.NotNil(t, actualMeta) compareDirMeta(expectedMeta, actualMeta, false) // test copying metadata from one dir to another copiedDir, err := operations.CopyDirMetadata(ctx, f, nil, "subdir2", newDst) assert.NoError(t, err) require.NotNil(t, copiedDir) assert.Equal(t, "subdir2", copiedDir.Remote()) actualMeta, err = fs.GetMetadata(ctx, copiedDir) assert.NoError(t, err) assert.NotNil(t, actualMeta) compareDirMeta(expectedMeta, actualMeta, true) // test DirModTimeUpdatesOnWrite expectedTime := copiedDir.ModTime(ctx) assert.True(t, !expectedTime.IsZero()) r.WriteObject(ctx, copiedDir.Remote()+"/"+randomFilename(), "hi there", t3) entries, err = f.List(ctx, "") assert.NoError(t, err) entries.ForDir(func(dir fs.Directory) { if dir.Remote() == copiedDir.Remote() { assert.True(t, expectedTime.Equal(dir.ModTime(ctx)), fmt.Sprintf("want %v got %v", expectedTime, dir.ModTime(ctx))) } }) } // TestServerSideCopyMove tests server-side Copy and Move func (f *Fs) TestServerSideCopyMove(t *testing.T, r *fstest.Run) { // setup ctx, ci := fs.AddConfig(ctx) ci.Metadata = true _ = f.opt.MetadataPermissions.Set("read,write") file1 := r.WriteFile(randomFilename(), content, t2) // add a permission with "read" role permissions := defaultPermissions(f.driveType) permissions[0].Roles[0] = api.ReadRole expectedMeta, actualMeta := f.putWithMeta(ctx, t, &file1, permissions) f.compareMeta(t, expectedMeta, actualMeta, false) comparePerms := func(expectedMeta, actualMeta fs.Metadata) (newExpectedMeta, newActualMeta fs.Metadata) { expectedP, actualP := unmarshalPerms(t, expectedMeta["permissions"]), unmarshalPerms(t, actualMeta["permissions"]) normalize(expectedP) normalize(actualP) expectedMeta.Set("permissions", marshalPerms(t, expectedP)) actualMeta.Set("permissions", marshalPerms(t, actualP)) assert.JSONEq(t, expectedMeta["permissions"], actualMeta["permissions"]) return expectedMeta, actualMeta } // Copy obj1, err := f.NewObject(ctx, file1.Path) assert.NoError(t, err) originalMeta := actualMeta obj2, err := f.Copy(ctx, obj1, randomFilename()) assert.NoError(t, err) actualMeta, err = fs.GetMetadata(ctx, obj2) assert.NoError(t, err) expectedMeta, actualMeta = comparePerms(originalMeta, actualMeta) f.compareMeta(t, expectedMeta, actualMeta, true) // Move obj3, err := f.Move(ctx, obj1, randomFilename()) assert.NoError(t, err) actualMeta, err = fs.GetMetadata(ctx, obj3) assert.NoError(t, err) expectedMeta, actualMeta = comparePerms(originalMeta, actualMeta) f.compareMeta(t, expectedMeta, actualMeta, true) } // TestMetadataMapper tests adding permissions with the --metadata-mapper func (f *Fs) TestMetadataMapper(t *testing.T, r *fstest.Run) { // setup ctx, ci := fs.AddConfig(ctx) ci.Metadata = true _ = f.opt.MetadataPermissions.Set("read,write") file1 := r.WriteFile(randomFilename(), content, t2) blob := `{"Metadata":{"permissions":"[{\"grantedToIdentities\":[{\"user\":{\"id\":\"ryan@contoso.com\"}}],\"roles\":[\"read\"]}]"}}` if f.driveType != driveTypePersonal { blob = `{"Metadata":{"permissions":"[{\"grantedToIdentitiesV2\":[{\"user\":{\"id\":\"ryan@contoso.com\"}}],\"roles\":[\"read\"]}]"}}` } // Copy ci.MetadataMapper = []string{"echo", blob} require.NoError(t, ci.Dump.Set("mapper")) obj1, err := r.Flocal.NewObject(ctx, file1.Path) assert.NoError(t, err) obj2, err := operations.Copy(ctx, f, nil, randomFilename(), obj1) assert.NoError(t, err) actualMeta, err := fs.GetMetadata(ctx, obj2) assert.NoError(t, err) actualP := unmarshalPerms(t, actualMeta["permissions"]) found := false foundCount := 0 for _, p := range actualP { for _, identity := range p.GetGrantedToIdentities(f.driveType) { if identity.User.DisplayName == testUserID { assert.Equal(t, []api.Role{api.ReadRole}, p.Roles) found = true foundCount++ } } if f.driveType == driveTypePersonal { if p.GetGrantedTo(f.driveType) != nil && p.GetGrantedTo(f.driveType).User != (api.Identity{}) && p.GetGrantedTo(f.driveType).User.ID == testUserID { // shows up in a different place on biz vs. personal assert.Equal(t, []api.Role{api.ReadRole}, p.Roles) found = true foundCount++ } } } assert.True(t, found, fmt.Sprintf("no permission found with expected role (want: \n\n%v \n\ngot: \n\n%v\n\n)", blob, actualMeta)) assert.Equal(t, 1, foundCount, "expected to find exactly 1 match") } // helper function to put an object with metadata and permissions func (f *Fs) putWithMeta(ctx context.Context, t *testing.T, file *fstest.Item, perms []*api.PermissionsType) (expectedMeta, actualMeta fs.Metadata) { t.Helper() expectedMeta = fs.Metadata{ "mtime": t1.Format(timeFormatOut), "btime": t2.Format(timeFormatOut), "description": "that is so meta!", } expectedMeta.Set("permissions", marshalPerms(t, perms)) obj := fstests.PutTestContentsMetadata(ctx, t, f, file, false, content, true, "plain/text", expectedMeta) do, ok := obj.(fs.Metadataer) require.True(t, ok) actualMeta, err := do.Metadata(ctx) require.NoError(t, err) return expectedMeta, actualMeta } func randomFilename() string { return "some file-" + random.String(8) + ".txt" } func (f *Fs) compareMeta(t *testing.T, expectedMeta, actualMeta fs.Metadata, ignoreID bool) { t.Helper() for k, v := range expectedMeta { gotV, ok := actualMeta[k] switch k { case "shared-owner-id", "shared-time", "shared-by-id", "shared-scope": continue case "permissions": continue case "utime": assert.True(t, ok, fmt.Sprintf("expected metadata key is missing: %v", k)) if f.driveType == driveTypePersonal { compareTimeStrings(t, k, v, gotV, time.Minute) // read-only upload time, so slight difference expected -- use larger precision continue } compareTimeStrings(t, k, expectedMeta["btime"], gotV, time.Minute) // another bizarre difference between personal and business... continue case "id": if ignoreID { continue // different id is expected when copying meta from one item to another } case "mtime", "btime": assert.True(t, ok, fmt.Sprintf("expected metadata key is missing: %v", k)) compareTimeStrings(t, k, v, gotV, time.Second) continue case "description": if f.driveType != driveTypePersonal { continue // not supported } } assert.True(t, ok, fmt.Sprintf("expected metadata key is missing: %v", k)) assert.Equal(t, v, gotV, actualMeta) } } func compareTimeStrings(t *testing.T, remote, want, got string, precision time.Duration) { wantT, err := time.Parse(timeFormatIn, want) assert.NoError(t, err) gotT, err := time.Parse(timeFormatIn, got) assert.NoError(t, err) fstest.AssertTimeEqualWithPrecision(t, remote, wantT, gotT, precision) } func marshalPerms(t *testing.T, p []*api.PermissionsType) string { b, err := json.MarshalIndent(p, "", "\t") assert.NoError(t, err) return string(b) } func unmarshalPerms(t *testing.T, perms string) (p []*api.PermissionsType) { t.Helper() err := json.Unmarshal([]byte(perms), &p) assert.NoError(t, err) return p } func indent(t *testing.T, s string) string { p := unmarshalPerms(t, s) return marshalPerms(t, p) } func defaultPermissions(driveType string) []*api.PermissionsType { if driveType == driveTypePersonal { return []*api.PermissionsType{{ GrantedTo: &api.IdentitySet{User: api.Identity{}}, GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{ID: testUserID}}}, Roles: []api.Role{api.WriteRole}, }} } return []*api.PermissionsType{{ GrantedToV2: &api.IdentitySet{User: api.Identity{}}, GrantedToIdentitiesV2: []*api.IdentitySet{{User: api.Identity{ID: testUserID}}}, Roles: []api.Role{api.WriteRole}, }} } // zeroes out some things we expect to be different when copying/moving between objects func normalize(Ps []*api.PermissionsType) { for _, ep := range Ps { ep.ID = "" ep.Link = nil ep.ShareID = "" } } func (f *Fs) resetTestDefaults(r *fstest.Run) { ci := fs.GetConfig(ctx) ci.Metadata = false _ = f.opt.MetadataPermissions.Set("off") r.Finalise() } // InternalTest dispatches all internal tests func (f *Fs) InternalTest(t *testing.T) { newTestF := func() (*Fs, *fstest.Run) { r := fstest.NewRunIndividual(t) testF, ok := r.Fremote.(*Fs) if !ok { t.FailNow() } return testF, r } testF, r := newTestF() t.Run("TestWritePermissions", func(t *testing.T) { testF.TestWritePermissions(t, r) }) testF.resetTestDefaults(r) testF, r = newTestF() t.Run("TestUploadSinglePart", func(t *testing.T) { testF.TestUploadSinglePart(t, r) }) testF.resetTestDefaults(r) testF, r = newTestF() t.Run("TestReadPermissions", func(t *testing.T) { testF.TestReadPermissions(t, r) }) testF.resetTestDefaults(r) testF, r = newTestF() t.Run("TestReadMetadata", func(t *testing.T) { testF.TestReadMetadata(t, r) }) testF.resetTestDefaults(r) testF, r = newTestF() t.Run("TestDirectoryMetadata", func(t *testing.T) { testF.TestDirectoryMetadata(t, r) }) testF.resetTestDefaults(r) testF, r = newTestF() t.Run("TestServerSideCopyMove", func(t *testing.T) { testF.TestServerSideCopyMove(t, r) }) testF.resetTestDefaults(r) t.Run("TestMetadataMapper", func(t *testing.T) { testF.TestMetadataMapper(t, r) }) testF.resetTestDefaults(r) } var _ fstests.InternalTester = (*Fs)(nil)