extern/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-12-03 13:55:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Initial commit - some small parts working

Browse Source
This commit is contained in:
Nick Craig-Wood 2012-11-18 17:32:31 +00:00
commit e9ae4f89a4
6 changed files with 377 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
*~
*.pyc
test-env*
junk/
swiftsync

20
COPYING Normal file
View File

@ -0,0 +1,20 @@
Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

53
README.md Normal file
View File

@ -0,0 +1,53 @@
Swiftsync
==========
Sync files and directories to and from swift
FIXME
Install
-------
Swiftsync is a Go program and comes as a single binary file.
Download the relevant binary from
- http://www.craig-wood.com/nick/pub/swiftsync/
Or alternatively if you have Go installed use
go get github.com/ncw/swiftsync
and this will build the binary in `$GOPATH/bin`. You can then modify
the source and submit patches.
Usage
-----
FIXME
License
-------
This is free software under the terms of MIT the license (check the
COPYING file included in this package).
Contact and support
-------------------
The project website is at:
- https://github.com/ncw/swiftsync
There you can file bug reports, ask for help or contribute patches.
Authors
-------
- Nick Craig-Wood <nick@craig-wood.com>
Contributors
------------
- Your name goes here!

32
notes.txt Normal file
View File

@ -0,0 +1,32 @@
make 100% compatible with swift.py?
Make Env vars compatible with st?
Get and put the metadata in the libray (x-object-meta-mtime) when getting and putting a file?
st is setting this
'x-object-meta-mtime'
getmtime(filename)
Return the last modification time of a file, reported by os.stat().
>>> f = os.path.getmtime("z")
1347717491.343554
>>> print f
1347717491.34
>>> str(f)
'1347717491.34'
>>> "%d" % f
'1347717491'
>>>
swift.py appears to be doing it wrong with str(float) which isn't a
good way of stringifying floats...
Make
This also puts meta-mtime
https://github.com/gholt/swiftly
As an integer, but it does parse it as a float
subargs.append('x-object-meta-mtime:%d' % getmtime(options.input_))

155
swiftsync.go Normal file
View File

@ -0,0 +1,155 @@
// Sync files and directories to and from swift
//
// Nick Craig-Wood <nick@craig-wood.com>
package main
import (
//"bytes"
"flag"
"fmt"
//"io"
//"io/ioutil"
"log"
//"math/rand"
"os"
//"os/signal"
//"path/filepath"
//"regexp"
//"runtime"
"runtime/pprof"
"strconv"
"strings"
//"sync"
//"syscall"
//"time"
"github.com/ncw/swift"
)
// Globals
var (
// Flags
//fileSize = flag.Int64("s", 1E9, "Size of the check files")
cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
//duration = flag.Duration("duration", time.Hour*24, "Duration to run test")
//statsInterval = flag.Duration("stats", time.Minute*1, "Interval to print stats")
//logfile = flag.String("logfile", "stressdisk.log", "File to write log to set to empty to ignore")
snet = flag.Bool("snet", false, "Use internal service network") // FIXME not implemented
verbose = flag.Bool("verbose", false, "Print lots more stuff")
quiet = flag.Bool("quiet", false, "Print as little stuff as possible")
// FIXME make these part of swift so we get a standard set of flags?
authUrl = flag.String("auth", os.Getenv("SWIFT_AUTH_USER"), "Auth URL for server. Defaults to environment var SWIFT_AUTH_USER.")
userName = flag.String("user", os.Getenv("ST_USER"), "User name. Defaults to environment var ST_USER.")
apiKey = flag.String("key", os.Getenv("ST_KEY"), "API key (password). Defaults to environment var ST_KEY.")
)
// Turns a number of ns into a floating point string in seconds
//
// Trims trailing zeros and guaranteed to be perfectly accurate
func nsToFloatString(ns int64) string {
if ns < 0 {
return "-" + nsToFloatString(-ns)
}
result := fmt.Sprintf("%010d", ns)
split := len(result) - 9
result, decimals := result[:split], result[split:]
decimals = strings.TrimRight(decimals, "0")
if decimals != "" {
result += "."
result += decimals
}
return result
}
// Turns a floating point string in seconds into a ns integer
//
// Guaranteed to be perfectly accurate
func floatStringToNs(s string) (ns int64, err error) {
if s != "" && s[0] == '-' {
ns, err = floatStringToNs(s[1:])
return -ns, err
}
point := strings.IndexRune(s, '.')
if point >= 0 {
tail := s[point+1:]
if len(tail) > 0 {
if len(tail) > 9 {
tail = tail[:9]
}
uns, err := strconv.ParseUint(tail, 10, 64)
if err != nil {
return 0, err
}
ns = int64(uns)
for i := 9 - len(tail); i > 0; i-- {
ns *= 10
}
}
s = s[:point]
}
secs, err := strconv.ParseInt(s, 10, 64)
if err != nil {
return 0, err
}
ns += int64(1000000000) * secs
return ns, nil
}
// syntaxError prints the syntax
func syntaxError() {
fmt.Fprintf(os.Stderr, `Sync files and directores to and from swift
FIXME
Full options:
`)
flag.PrintDefaults()
}
// Exit with the message
func fatal(message string, args ...interface{}) {
syntaxError()
fmt.Fprintf(os.Stderr, message, args...)
os.Exit(1)
}
func main() {
flag.Usage = syntaxError
flag.Parse()
//args := flag.Args()
//runtime.GOMAXPROCS(3)
// Setup profiling if desired
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
// if len(args) < 1 {
// fatal("No command supplied\n")
// }
if *userName == "" {
log.Fatal("Need --user or environmental variable ST_USER")
}
if *apiKey == "" {
log.Fatal("Need --key or environmental variable ST_KEY")
}
if *authUrl == "" {
log.Fatal("Need --auth or environmental variable ST_AUTH")
}
c := swift.Connection{
UserName: *userName,
ApiKey: *apiKey,
AuthUrl: *authUrl,
}
err := c.Authenticate()
if err != nil {
log.Fatal("Failed to authenticate", err)
}
}

112
swiftsync_test.go Normal file
View File

@ -0,0 +1,112 @@
// Tests for swiftsync
package main
import (
"testing"
)
func TestNsToFloatString(t *testing.T) {
for _, d := range []struct {
ns int64
fs string
}{
{0, "0"},
{1, "0.000000001"},
{1000, "0.000001"},
{1000000, "0.001"},
{100000000, "0.1"},
{1000000000, "1"},
{10000000000, "10"},
{12345678912, "12.345678912"},
{12345678910, "12.34567891"},
{12345678900, "12.3456789"},
{12345678000, "12.345678"},
{12345670000, "12.34567"},
{12345600000, "12.3456"},
{12345000000, "12.345"},
{12340000000, "12.34"},
{12300000000, "12.3"},
{12000000000, "12"},
{10000000000, "10"},
{1347717491123123123, "1347717491.123123123"},
} {
if nsToFloatString(d.ns) != d.fs {
t.Error("Failed", d.ns, "!=", d.fs)
}
if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs {
t.Error("Failed on negative", d.ns, "!=", d.fs)
}
}
}
func TestFloatStringToNs(t *testing.T) {
for _, d := range []struct {
ns int64
fs string
}{
{0, "0"},
{0, "0."},
{0, "0.0"},
{0, "0.0000000001"},
{1, "0.000000001"},
{1000, "0.000001"},
{1000000, "0.001"},
{100000000, "0.1"},
{100000000, "0.10"},
{100000000, "0.1000000001"},
{1000000000, "1"},
{1000000000, "1."},
{1000000000, "1.0"},
{10000000000, "10"},
{12345678912, "12.345678912"},
{12345678912, "12.3456789129"},
{12345678912, "12.34567891299"},
{12345678910, "12.34567891"},
{12345678900, "12.3456789"},
{12345678000, "12.345678"},
{12345670000, "12.34567"},
{12345600000, "12.3456"},
{12345000000, "12.345"},
{12340000000, "12.34"},
{12300000000, "12.3"},
{12000000000, "12"},
{10000000000, "10"},
// This is a typical value which has more bits in than a float64
{1347717491123123123, "1347717491.123123123"},
} {
ns, err := floatStringToNs(d.fs)
if err != nil {
t.Error("Failed conversion", err)
}
if ns != d.ns {
t.Error("Failed", d.fs, "!=", d.ns, "was", ns)
}
if d.ns > 0 {
ns, err := floatStringToNs("-" + d.fs)
if err != nil {
t.Error("Failed conversion", err)
}
if ns != -d.ns {
t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs)
}
}
}
// These are expected to produce errors
for _, fs := range []string{
"",
".0",
" 1",
"- 1",
"- 1",
"1.-1",
"1.0.0",
"1x0",
} {
ns, err := floatStringToNs(fs)
if err == nil {
t.Error("Didn't produce expected error", fs, ns)
}
}
}