rclone ncdu for exploring a remote with a text based user interface.

cmd/all/all.go

@@ -30,6 +30,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/mount"
 	_ "github.com/ncw/rclone/cmd/move"
 	_ "github.com/ncw/rclone/cmd/moveto"
+	_ "github.com/ncw/rclone/cmd/ncdu"
 	_ "github.com/ncw/rclone/cmd/obscure"
 	_ "github.com/ncw/rclone/cmd/purge"
 	_ "github.com/ncw/rclone/cmd/rmdir"

cmd/ncdu/ncdu.go

@@ -0,0 +1,542 @@
+package ncdu
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/ncdu/scan"
+	"github.com/ncw/rclone/fs"
+	termbox "github.com/nsf/termbox-go"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "ncdu remote:path",
+	Short: `Explore a remote with a text based user interface.`,
+	Long: `
+This displays a text based user interface allowing the navigation of a
+remote. It is most useful for answering the question - "What is using
+all my disk space?".
+
+To make the user interface it first scans the entire remote given and
+builds an in memory representation.  rclone ncdu can be used during
+this scanning phase and you will see it building up the directory
+structure as it goes along.
+
+Here are the keys - press '?' to toggle the help on and off
+
+    ` + strings.Join(helpText[1:], "\n    ") + `
+
+This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
+rclone remotes.  It is missing lots of features at the moment, most
+importantly deleting files, but is useful as it stands.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		fsrc := cmd.NewFsSrc(args)
+		cmd.Run(false, false, command, func() error {
+			return NewUI(fsrc).Show()
+		})
+	},
+}
+
+// help text
+var helpText = []string{
+	"rclone ncdu",
+	" ↑,↓ or k,j to Move",
+	" →,l to enter",
+	" ←,h to return",
+	" c toggle counts",
+	" g toggle graph",
+	" n,s,C sort by name,size,count",
+	" ? to toggle help on and off",
+	" q/ESC/c-C to quit",
+}
+
+// UI contains the state of the user interface
+type UI struct {
+	f             fs.Fs         // fs being displayed
+	fsName        string        // human name of Fs
+	root          *scan.Dir     // root directory
+	d             *scan.Dir     // current directory being displayed
+	path          string        // path of current directory
+	showBox       bool          // whether to show a box
+	boxText       []string      // text to show in box
+	entries       fs.DirEntries // entries of current directory
+	sortPerm      []int         // order to display entries in after sorting
+	invSortPerm   []int         // inverse order
+	dirListHeight int           // height of listing
+	listing       bool          // whether listing is in progress
+	showGraph     bool          // toggle showing graph
+	showCounts    bool          // toggle showing counts
+	sortByName    int8          // +1 for normal, 0 for off, -1 for reverse
+	sortBySize    int8
+	sortByCount   int8
+	dirPosMap     map[string]dirPos // store for directory positions
+}
+
+// Where we have got to in the directory listing
+type dirPos struct {
+	entry  int
+	offset int
+}
+
+// Print a string
+func Print(x, y int, fg, bg termbox.Attribute, msg string) {
+	for _, c := range msg {
+		termbox.SetCell(x, y, c, fg, bg)
+		x++
+	}
+}
+
+// Printf a string
+func Printf(x, y int, fg, bg termbox.Attribute, format string, args ...interface{}) {
+	s := fmt.Sprintf(format, args...)
+	Print(x, y, fg, bg, s)
+}
+
+// Line prints a string to given xmax, with given space
+func Line(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, msg string) {
+	for _, c := range msg {
+		termbox.SetCell(x, y, c, fg, bg)
+		x++
+		if x >= xmax {
+			return
+		}
+	}
+	for ; x < xmax; x++ {
+		termbox.SetCell(x, y, spacer, fg, bg)
+	}
+}
+
+// Linef a string
+func Linef(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, format string, args ...interface{}) {
+	s := fmt.Sprintf(format, args...)
+	Line(x, y, xmax, fg, bg, spacer, s)
+}
+
+// Box the u.boxText onto the screen
+func (u *UI) Box() {
+	w, h := termbox.Size()
+
+	// Find dimensions of text
+	boxWidth := 10
+	for _, s := range u.boxText {
+		if len(s) > boxWidth && len(s) < w-4 {
+			boxWidth = len(s)
+		}
+	}
+	boxHeight := len(u.boxText)
+
+	// position
+	x := (w - boxWidth) / 2
+	y := (h - boxHeight) / 2
+	xmax := x + boxWidth
+
+	// draw text
+	fg, bg := termbox.ColorRed, termbox.ColorWhite
+	for i, s := range u.boxText {
+		Line(x, y+i, xmax, fg, bg, ' ', s)
+		fg = termbox.ColorBlack
+	}
+
+	// FIXME draw a box around
+}
+
+// find the biggest entry in the current listing
+func (u *UI) biggestEntry() (biggest int64) {
+	if u.d == nil {
+		return
+	}
+	for i := range u.entries {
+		size, _, _, _ := u.d.AttrI(u.sortPerm[i])
+		if size > biggest {
+			biggest = size
+		}
+	}
+	return
+}
+
+// Draw the current screen
+func (u *UI) Draw() error {
+	w, h := termbox.Size()
+	u.dirListHeight = h - 3
+
+	// Plot
+	err := termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
+	if err != nil {
+		return errors.Wrap(err, "failed to clear screen")
+	}
+
+	// Header line
+	Linef(0, 0, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "rclone ncdu %s - use the arrow keys to navigate, press ? for help", fs.Version)
+
+	// Directory line
+	Linef(0, 1, w, termbox.ColorWhite, termbox.ColorBlack, '-', "-- %s ", u.path)
+
+	// graphs
+	const (
+		graphBars = 10
+		graph     = "##########          "
+	)
+
+	// Directory listing
+	if u.d != nil {
+		y := 2
+		perBar := u.biggestEntry() / graphBars
+		if perBar == 0 {
+			perBar = 1
+		}
+		dirPos := u.dirPosMap[u.path]
+		for i, j := range u.sortPerm[dirPos.offset:] {
+			entry := u.entries[j]
+			n := i + dirPos.offset
+			if y >= h-1 {
+				break
+			}
+			fg := termbox.ColorWhite
+			bg := termbox.ColorBlack
+			if n == dirPos.entry {
+				fg, bg = bg, fg
+			}
+			size, count, isDir, readable := u.d.AttrI(u.sortPerm[n])
+			mark := ' '
+			if isDir {
+				mark = '/'
+			}
+			message := ""
+			if !readable {
+				message = " [not read yet]"
+			}
+			extras := ""
+			if u.showCounts {
+				if count > 0 {
+					extras += fmt.Sprintf("%8v ", fs.SizeSuffix(count))
+				} else {
+					extras += "         "
+				}
+
+			}
+			if u.showGraph {
+				bars := (size + perBar/2 - 1) / perBar
+				// clip if necessary - only happens during startup
+				if bars > 10 {
+					bars = 10
+				} else if bars < 0 {
+					bars = 0
+				}
+				extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
+			}
+			Linef(0, y, w, fg, bg, ' ', "%8v %s%c%s%s", fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
+			y++
+		}
+	}
+
+	// Footer
+	if u.d == nil {
+		Line(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Waiting for root directory...")
+	} else {
+		message := ""
+		if u.listing {
+			message = " [listing in progress]"
+		}
+		size, count := u.d.Attr()
+		Linef(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Total usage: %v, Objects: %d%s", fs.SizeSuffix(size), count, message)
+	}
+
+	// Show the box on top if requred
+	if u.showBox {
+		u.Box()
+	}
+	err = termbox.Flush()
+	if err != nil {
+		return errors.Wrap(err, "failed to flush screen")
+	}
+	return nil
+}
+
+// Move the cursor this many spaces adjusting the viewport as necessary
+func (u *UI) move(d int) {
+	if u.d == nil {
+		return
+	}
+
+	absD := d
+	if d < 0 {
+		absD = -d
+	}
+
+	entries := len(u.entries)
+
+	// Fetch current dirPos
+	dirPos := u.dirPosMap[u.path]
+
+	dirPos.entry += d
+
+	// check entry in range
+	if dirPos.entry < 0 {
+		dirPos.entry = 0
+	} else if dirPos.entry >= entries {
+		dirPos.entry = entries - 1
+	}
+
+	// check cursor still on screen
+	p := dirPos.entry - dirPos.offset // where dirPos.entry appears on the screen
+	if p < 0 {
+		dirPos.offset -= absD
+	} else if p >= u.dirListHeight {
+		dirPos.offset += absD
+	}
+
+	// check dirPos.offset in bounds
+	if dirPos.offset < 0 {
+		dirPos.offset = 0
+	} else if dirPos.offset >= entries {
+		dirPos.offset = entries - 1
+	}
+
+	// write dirPos back for later
+	u.dirPosMap[u.path] = dirPos
+}
+
+// Sort by the configured sort method
+type ncduSort struct {
+	sortPerm []int
+	entries  fs.DirEntries
+	d        *scan.Dir
+	u        *UI
+}
+
+// Less is part of sort.Interface.
+func (ds *ncduSort) Less(i, j int) bool {
+	isize, icount, _, _ := ds.d.AttrI(ds.sortPerm[i])
+	jsize, jcount, _, _ := ds.d.AttrI(ds.sortPerm[j])
+	iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
+	switch {
+	case ds.u.sortByName < 0:
+		return iname > jname
+	case ds.u.sortByName > 0:
+		break
+	case ds.u.sortBySize < 0:
+		if isize != jsize {
+			return isize < jsize
+		}
+	case ds.u.sortBySize > 0:
+		if isize != jsize {
+			return isize > jsize
+		}
+	case ds.u.sortByCount < 0:
+		if icount != jcount {
+			return icount < jcount
+		}
+	case ds.u.sortByCount > 0:
+		if icount != jcount {
+			return icount > jcount
+		}
+	}
+	// if everything equal, sort by name
+	return iname < jname
+}
+
+// Swap is part of sort.Interface.
+func (ds *ncduSort) Swap(i, j int) {
+	ds.sortPerm[i], ds.sortPerm[j] = ds.sortPerm[j], ds.sortPerm[i]
+}
+
+// Len is part of sort.Interface.
+func (ds *ncduSort) Len() int {
+	return len(ds.sortPerm)
+}
+
+// sort the permutation map of the current directory
+func (u *UI) sortCurrentDir() {
+	u.sortPerm = u.sortPerm[:0]
+	for i := range u.entries {
+		u.sortPerm = append(u.sortPerm, i)
+	}
+	data := ncduSort{
+		sortPerm: u.sortPerm,
+		entries:  u.entries,
+		d:        u.d,
+		u:        u,
+	}
+	sort.Sort(&data)
+	if len(u.invSortPerm) < len(u.sortPerm) {
+		u.invSortPerm = make([]int, len(u.sortPerm))
+	}
+	for i, j := range u.sortPerm {
+		u.invSortPerm[j] = i
+	}
+}
+
+// setCurrentDir sets the current directory
+func (u *UI) setCurrentDir(d *scan.Dir) {
+	u.d = d
+	u.entries = d.Entries()
+	u.path = path.Join(u.fsName, d.Path())
+	u.sortCurrentDir()
+}
+
+// enters the current entry
+func (u *UI) enter() {
+	if u.d == nil {
+		return
+	}
+	dirPos := u.dirPosMap[u.path]
+	d, _ := u.d.GetDir(u.sortPerm[dirPos.entry])
+	if d == nil {
+		return
+	}
+	u.setCurrentDir(d)
+}
+
+// up goes up to the parent directory
+func (u *UI) up() {
+	if u.d == nil {
+		return
+	}
+	parent := u.d.Parent()
+	if parent != nil {
+		u.setCurrentDir(parent)
+	}
+}
+
+// popupBox shows a box with the text in
+func (u *UI) popupBox(text []string) {
+	u.boxText = text
+	u.showBox = true
+}
+
+// togglePopupBox shows a box with the text in
+func (u *UI) togglePopupBox(text []string) {
+	if u.showBox {
+		u.showBox = false
+	} else {
+		u.popupBox(text)
+	}
+}
+
+// toggle the sorting for the flag passed in
+func (u *UI) toggleSort(sortType *int8) {
+	old := *sortType
+	u.sortBySize = 0
+	u.sortByCount = 0
+	u.sortByName = 0
+	if old == 0 {
+		*sortType = 1
+	} else {
+		*sortType = -old
+	}
+	u.sortCurrentDir()
+}
+
+// NewUI creates a new user interface for ncdu on f
+func NewUI(f fs.Fs) *UI {
+	return &UI{
+		f:             f,
+		path:          "Waiting for root...",
+		dirListHeight: 20, // updated in Draw
+		fsName:        f.Name() + ":" + f.Root(),
+		showGraph:     true,
+		showCounts:    false,
+		sortByName:    0, // +1 for normal, 0 for off, -1 for reverse
+		sortBySize:    1,
+		sortByCount:   0,
+		dirPosMap:     make(map[string]dirPos),
+	}
+}
+
+// Show shows the user interface
+func (u *UI) Show() error {
+	err := termbox.Init()
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer termbox.Close()
+
+	// scan the disk in the background
+	u.listing = true
+	rootChan, errChan, updated := scan.Scan(u.f)
+
+	// Poll the events into a channel
+	events := make(chan termbox.Event)
+	doneWithEvent := make(chan bool)
+	go func() {
+		for {
+			events <- termbox.PollEvent()
+			<-doneWithEvent
+		}
+	}()
+
+	// Main loop, waiting for events and channels
+outer:
+	for {
+		//Reset()
+		err := u.Draw()
+		if err != nil {
+			return errors.Wrap(err, "draw failed")
+		}
+		var root *scan.Dir
+		select {
+		case root = <-rootChan:
+			u.root = root
+			u.setCurrentDir(root)
+		case err := <-errChan:
+			if err != nil {
+				return errors.Wrap(err, "ncdu directory listing")
+			}
+			u.listing = false
+		case <-updated:
+			// redraw
+			// might want to limit updates per second
+			u.sortCurrentDir()
+		case ev := <-events:
+			doneWithEvent <- true
+			if ev.Type == termbox.EventKey {
+				switch ev.Key + termbox.Key(ev.Ch) {
+				case termbox.KeyEsc, termbox.KeyCtrlC, 'q':
+					if u.showBox {
+						u.showBox = false
+					} else {
+						break outer
+					}
+				case termbox.KeyArrowDown, 'j':
+					u.move(1)
+				case termbox.KeyArrowUp, 'k':
+					u.move(-1)
+				case termbox.KeyPgdn, '-', '_':
+					u.move(u.dirListHeight)
+				case termbox.KeyPgup, '=', '+':
+					u.move(-u.dirListHeight)
+				case termbox.KeyArrowLeft, 'h':
+					u.up()
+				case termbox.KeyArrowRight, 'l', termbox.KeyEnter:
+					u.enter()
+				case 'c':
+					u.showCounts = !u.showCounts
+				case 'g':
+					u.showGraph = !u.showGraph
+				case 'n':
+					u.toggleSort(&u.sortByName)
+				case 's':
+					u.toggleSort(&u.sortBySize)
+				case 'C':
+					u.toggleSort(&u.sortByCount)
+				case '?':
+					u.togglePopupBox(helpText)
+				}
+			}
+		}
+		// listen to key presses, etc
+	}
+	return nil
+}

cmd/ncdu/scan/scan.go

@@ -0,0 +1,167 @@
+// Package scan does concurrent scanning of an Fs building up a directory tree.
+package scan
+
+import (
+	"path"
+	"sync"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+// Dir represents a directory found in the remote
+type Dir struct {
+	parent   *Dir
+	path     string
+	mu       sync.Mutex
+	count    int64
+	size     int64
+	complete bool
+	entries  fs.DirEntries
+	dirs     map[string]*Dir
+	offset   int // current listing offset
+	entry    int // current listing entry
+}
+
+// Parent returns the directory above this one
+func (d *Dir) Parent() *Dir {
+	// no locking needed since these are write once in newDir()
+	return d.parent
+}
+
+// Path returns the position of the dir in the filesystem
+func (d *Dir) Path() string {
+	// no locking needed since these are write once in newDir()
+	return d.path
+}
+
+// make a new directory
+func newDir(parent *Dir, dirPath string, entries fs.DirEntries) *Dir {
+	d := &Dir{
+		parent:  parent,
+		path:    dirPath,
+		entries: entries,
+		dirs:    make(map[string]*Dir),
+	}
+	// Count size in this dir
+	for _, entry := range entries {
+		if o, ok := entry.(fs.Object); ok {
+			d.count++
+			d.size += o.Size()
+		}
+	}
+	// Set my directory entry in parent
+	if parent != nil {
+		parent.mu.Lock()
+		leaf := path.Base(dirPath)
+		d.parent.dirs[leaf] = d
+		parent.mu.Unlock()
+	}
+	// Accumulate counts in parents
+	for ; parent != nil; parent = parent.parent {
+		parent.mu.Lock()
+		parent.count += d.count
+		parent.size += d.size
+		parent.mu.Unlock()
+	}
+	return d
+}
+
+// Entries returns a copy of the entries in the directory
+func (d *Dir) Entries() fs.DirEntries {
+	return append(fs.DirEntries(nil), d.entries...)
+}
+
+// gets the directory of the i-th entry
+//
+// returns nil if it is a file
+// returns a flag as to whether is directory or not
+//
+// Call with d.mu held
+func (d *Dir) getDir(i int) (subDir *Dir, isDir bool) {
+	obj := d.entries[i]
+	dir, ok := obj.(*fs.Dir)
+	if !ok {
+		return nil, false
+	}
+	leaf := path.Base(dir.Remote())
+	subDir = d.dirs[leaf]
+	return subDir, true
+}
+
+// GetDir returns the Dir of the i-th entry
+//
+// returns nil if it is a file
+// returns a flag as to whether is directory or not
+func (d *Dir) GetDir(i int) (subDir *Dir, isDir bool) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.getDir(i)
+}
+
+// Attr returns the size and count for the directory
+func (d *Dir) Attr() (size int64, count int64) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.size, d.count
+}
+
+// AttrI returns the size, count and flags for the i-th directory entry
+func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	subDir, isDir := d.getDir(i)
+	if !isDir {
+		return d.entries[i].Size(), 0, false, true
+	}
+	if subDir == nil {
+		return 0, 0, true, false
+	}
+	size, count = subDir.Attr()
+	return size, count, true, true
+}
+
+// Scan the Fs passed in, returning a root directory channel and an
+// error channel
+func Scan(f fs.Fs) (chan *Dir, chan error, chan struct{}) {
+	root := make(chan *Dir, 1)
+	errChan := make(chan error, 1)
+	updated := make(chan struct{}, 1)
+	go func() {
+		parents := map[string]*Dir{}
+		err := fs.Walk(f, "", false, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
+			if err != nil {
+				return err // FIXME mark directory as errored instead of aborting
+			}
+			var parent *Dir
+			if dirPath != "" {
+				parentPath := path.Dir(dirPath)
+				if parentPath == "." {
+					parentPath = ""
+				}
+				var ok bool
+				parent, ok = parents[parentPath]
+				if !ok {
+					errChan <- errors.Errorf("couldn't find parent for %q", dirPath)
+				}
+			}
+			d := newDir(parent, dirPath, entries)
+			parents[dirPath] = d
+			if dirPath == "" {
+				root <- d
+			}
+			// Mark updated
+			select {
+			case updated <- struct{}{}:
+			default:
+				break
+			}
+			return nil
+		})
+		if err != nil {
+			errChan <- errors.Wrap(err, "ncdu listing failed")
+		}
+		errChan <- nil
+	}()
+	return root, errChan, updated
+}