package lsjson import ( "encoding/json" "fmt" "log" "os" "path" "time" "github.com/ncw/rclone/backend/crypt" "github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd/ls/lshelp" "github.com/ncw/rclone/fs" "github.com/ncw/rclone/fs/operations" "github.com/ncw/rclone/fs/walk" "github.com/pkg/errors" "github.com/spf13/cobra" ) var ( recurse bool showHash bool showEncrypted bool showOrigIDs bool noModTime bool ) func init() { cmd.Root.AddCommand(commandDefintion) commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.") commandDefintion.Flags().BoolVarP(&showHash, "hash", "", false, "Include hashes in the output (may take longer).") commandDefintion.Flags().BoolVarP(&noModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).") commandDefintion.Flags().BoolVarP(&showEncrypted, "encrypted", "M", false, "Show the encrypted names.") commandDefintion.Flags().BoolVarP(&showOrigIDs, "original", "", false, "Show the ID of the underlying Object.") } // lsJSON in the struct which gets marshalled for each line type lsJSON struct { Path string Name string Encrypted string `json:",omitempty"` Size int64 MimeType string `json:",omitempty"` ModTime Timestamp //`json:",omitempty"` IsDir bool Hashes map[string]string `json:",omitempty"` ID string `json:",omitempty"` OrigID string `json:",omitempty"` } // Timestamp a time in RFC3339 format with Nanosecond precision secongs type Timestamp time.Time // MarshalJSON turns a Timestamp into JSON func (t Timestamp) MarshalJSON() (out []byte, err error) { tt := time.Time(t) if tt.IsZero() { return []byte(`""`), nil } return []byte(`"` + tt.Format(time.RFC3339Nano) + `"`), nil } var commandDefintion = &cobra.Command{ Use: "lsjson remote:path", Short: `List directories and objects in the path in JSON format.`, Long: `List directories and objects in the path in JSON format. The output is an array of Items, where each Item looks like this { "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f", "MD5" : "b1946ac92492d2347c6235b4d2611184", "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" }, "ID": "y2djkhiujf83u33", "OrigID": "UYOJVTUW00Q1RzTDA", "IsDir" : false, "MimeType" : "application/octet-stream", "ModTime" : "2017-05-31T16:15:57.034468261+01:00", "Name" : "file.txt", "Encrypted" : "v0qpsdq8anpci8n929v3uu9338", "Path" : "full/path/goes/here/file.txt", "Size" : 6 } If --hash is not specified the Hashes property won't be emitted. If --no-modtime is specified then ModTime will be blank. If --encrypted is not specified the Encrypted won't be emitted. The Path field will only show folders below the remote path being listed. If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt" will be "subfolder/file.txt", not "remote:path/subfolder/file.txt". When used without --recursive the Path will always be the same as Name. The time is in RFC3339 format with nanosecond precision. The whole output can be processed as a JSON blob, or alternatively it can be processed line by line as each item is written one to a line. ` + lshelp.Help, Run: func(command *cobra.Command, args []string) { cmd.CheckArgs(1, 1, command, args) fsrc := cmd.NewFsSrc(args) var cipher crypt.Cipher if showEncrypted { fsInfo, _, _, config, err := fs.ConfigFs(args[0]) if err != nil { log.Fatalf(err.Error()) } if fsInfo.Name != "crypt" { log.Fatalf("The remote needs to be of type \"crypt\"") } cipher, err = crypt.NewCipher(config) if err != nil { log.Fatalf(err.Error()) } } cmd.Run(false, false, command, func() error { fmt.Println("[") first := true err := walk.Walk(fsrc, "", false, operations.ConfigMaxDepth(recurse), func(dirPath string, entries fs.DirEntries, err error) error { if err != nil { fs.CountError(err) fs.Errorf(dirPath, "error listing: %v", err) return nil } for _, entry := range entries { item := lsJSON{ Path: entry.Remote(), Name: path.Base(entry.Remote()), Size: entry.Size(), MimeType: fs.MimeTypeDirEntry(entry), } if !noModTime { item.ModTime = Timestamp(entry.ModTime()) } if cipher != nil { switch entry.(type) { case fs.Directory: item.Encrypted = cipher.EncryptDirName(path.Base(entry.Remote())) case fs.Object: item.Encrypted = cipher.EncryptFileName(path.Base(entry.Remote())) default: fs.Errorf(nil, "Unknown type %T in listing", entry) } } if do, ok := entry.(fs.IDer); ok { item.ID = do.ID() } if showOrigIDs { cur := entry for { u, ok := cur.(fs.ObjectUnWrapper) if !ok { break // not a wrapped object, use current id } next := u.UnWrap() if next == nil { break // no base object found, use current id } cur = next } if do, ok := cur.(fs.IDer); ok { item.OrigID = do.ID() } } switch x := entry.(type) { case fs.Directory: item.IsDir = true case fs.Object: item.IsDir = false if showHash { item.Hashes = make(map[string]string) for _, hashType := range x.Fs().Hashes().Array() { hash, err := x.Hash(hashType) if err != nil { fs.Errorf(x, "Failed to read hash: %v", err) } else if hash != "" { item.Hashes[hashType.String()] = hash } } } default: fs.Errorf(nil, "Unknown type %T in listing", entry) } out, err := json.Marshal(item) if err != nil { return errors.Wrap(err, "failed to marshal list object") } if first { first = false } else { fmt.Print(",\n") } _, err = os.Stdout.Write(out) if err != nil { return errors.Wrap(err, "failed to write to output") } } return nil }) if err != nil { return errors.Wrap(err, "error listing JSON") } if !first { fmt.Println() } fmt.Println("]") return nil }) }, }