mirror of
https://github.com/rclone/rclone.git
synced 2025-01-08 07:20:42 +01:00
press: first overhaul
... fix error handling for metadata upload ... fix metadata wrapping ... make sure not to leave any orphaned data when overwriting existing files with Put, Copy or Move ... switch from id01/go-lz4 to pierrec/lz4 ... use klauspost/compress/gzip for gzip ... use ulikunitz/xz for xz ... remove snappy
This commit is contained in:
buengese
parent
e41a88fb23
commit
ffe97865a7
@ -28,6 +28,7 @@ import (
|
||||
_ "github.com/rclone/rclone/backend/opendrive"
|
||||
_ "github.com/rclone/rclone/backend/pcloud"
|
||||
_ "github.com/rclone/rclone/backend/premiumizeme"
|
||||
_ "github.com/rclone/rclone/backend/press"
|
||||
_ "github.com/rclone/rclone/backend/putio"
|
||||
_ "github.com/rclone/rclone/backend/qingstor"
|
||||
_ "github.com/rclone/rclone/backend/s3"
|
||||
|
||||
@ -1,98 +0,0 @@
|
||||
package press
|
||||
|
||||
// This file implements shell exec algorithms that require binaries.
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// XZ command
|
||||
const xzcommand = "xz" // Name of xz binary (if available)
|
||||
|
||||
// ExecHeader - Header we add to an exec file. We don't need this.
|
||||
var ExecHeader = []byte{}
|
||||
|
||||
// Function that checks whether XZ is present in the system
|
||||
func checkXZ() bool {
|
||||
_, err := exec.LookPath("xz")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Function that gets binary paths if needed
|
||||
func getBinPaths(c *Compression, mode int) (err error) {
|
||||
err = nil
|
||||
if mode == XZMin || mode == XZDefault {
|
||||
c.BinPath, err = exec.LookPath(xzcommand)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Function that compresses a block using a shell command without wrapping in gzip. Requires an binary corresponding with the command.
|
||||
func (c *Compression) compressBlockExec(in []byte, out io.Writer, binaryPath string, args []string) (compressedSize uint32, uncompressedSize int64, err error) {
|
||||
// Initialize compression subprocess
|
||||
subprocess := exec.Command(binaryPath, args...)
|
||||
stdin, err := subprocess.StdinPipe()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Run subprocess that creates compressed file
|
||||
stdinError := make(chan error)
|
||||
go func() {
|
||||
_, err := stdin.Write(in)
|
||||
_ = stdin.Close()
|
||||
stdinError <- err
|
||||
}()
|
||||
|
||||
// Get output
|
||||
output, err := subprocess.Output()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Copy over
|
||||
n, err := io.Copy(out, bytes.NewReader(output))
|
||||
if err != nil {
|
||||
return uint32(n), int64(len(in)), err
|
||||
}
|
||||
|
||||
// Check if there was an error and return
|
||||
err = <-stdinError
|
||||
|
||||
return uint32(n), int64(len(in)), err
|
||||
}
|
||||
|
||||
// Utility function to decompress a block range using a shell command which wasn't wrapped in gzip
|
||||
func decompressBlockRangeExec(in io.Reader, out io.Writer, binaryPath string, args []string) (n int, err error) {
|
||||
// Decompress actual compression
|
||||
// Initialize decompression subprocess
|
||||
subprocess := exec.Command(binaryPath, args...)
|
||||
stdin, err := subprocess.StdinPipe()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Run subprocess that copies over compressed block
|
||||
stdinError := make(chan error)
|
||||
go func() {
|
||||
_, err := io.Copy(stdin, in)
|
||||
_ = stdin.Close()
|
||||
stdinError <- err
|
||||
}()
|
||||
|
||||
// Get output, copy, and return
|
||||
output, err := subprocess.Output()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n64, err := io.Copy(out, bytes.NewReader(output))
|
||||
if err != nil {
|
||||
return int(n64), err
|
||||
}
|
||||
err = <-stdinError
|
||||
return int(n64), err
|
||||
}
|
||||
@ -1,22 +1,48 @@
|
||||
package press
|
||||
|
||||
// This file implements the gzip algorithm.
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/gzip"
|
||||
)
|
||||
|
||||
// GzipHeader - Header we add to a gzip file. We're contatenating GZIP files here, so we don't need this.
|
||||
var GzipHeader = []byte{}
|
||||
// AlgGzip represents gzip compression algorithm
|
||||
type AlgGzip struct {
|
||||
level int
|
||||
blockSize uint32
|
||||
}
|
||||
|
||||
// Function that compresses a block using gzip
|
||||
func (c *Compression) compressBlockGz(in []byte, out io.Writer, compressionLevel int) (compressedSize uint32, uncompressedSize int64, err error) {
|
||||
// InitializeGzip initializes the gzip compression Algorithm
|
||||
func InitializeGzip(bs uint32, level int) Algorithm {
|
||||
a := new(AlgGzip)
|
||||
a.blockSize = bs
|
||||
a.level = level
|
||||
return a
|
||||
}
|
||||
|
||||
// GetFileExtension returns file extension
|
||||
func (a *AlgGzip) GetFileExtension() string {
|
||||
return ".gz"
|
||||
}
|
||||
|
||||
// GetHeader returns the Lz4 compression header
|
||||
func (a *AlgGzip) GetHeader() []byte {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
// GetFooter returns
|
||||
func (a *AlgGzip) GetFooter() []byte {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
// CompressBlock that compresses a block using gzip
|
||||
func (a *AlgGzip) CompressBlock(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize uint64, err error) {
|
||||
// Initialize buffer
|
||||
bufw := bufio.NewWriterSize(out, int(c.maxCompressedBlockSize()))
|
||||
bufw := bufio.NewWriterSize(out, int(a.blockSize+(a.blockSize)>>4))
|
||||
|
||||
// Initialize block writer
|
||||
outw, err := gzip.NewWriterLevel(bufw, compressionLevel)
|
||||
outw, err := gzip.NewWriterLevel(bufw, a.level)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
@ -35,11 +61,11 @@ func (c *Compression) compressBlockGz(in []byte, out io.Writer, compressionLevel
|
||||
blockSize := uint32(bufw.Buffered())
|
||||
err = bufw.Flush()
|
||||
|
||||
return blockSize, int64(len(in)), err
|
||||
return blockSize, uint64(len(in)), err
|
||||
}
|
||||
|
||||
// Utility function to decompress a block range using gzip
|
||||
func decompressBlockRangeGz(in io.Reader, out io.Writer) (n int, err error) {
|
||||
// DecompressBlock decompresses Lz4 compressed block
|
||||
func (a *AlgGzip) DecompressBlock(in io.Reader, out io.Writer, BlockSize uint32) (n int, err error) {
|
||||
gzipReader, err := gzip.NewReader(in)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
||||
@ -4,11 +4,12 @@ package press
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/bits"
|
||||
|
||||
"github.com/OneOfOne/xxhash"
|
||||
lz4 "github.com/id01/go-lz4"
|
||||
"github.com/buengese/xxh32"
|
||||
lz4 "github.com/pierrec/lz4"
|
||||
)
|
||||
|
||||
/*
|
||||
@ -31,65 +32,192 @@ Header checksum byte (xxhash(flags and bd byte) >> 1) & 0xff
|
||||
*/
|
||||
|
||||
// LZ4Header - Header of our LZ4 file
|
||||
var LZ4Header = []byte{0x04, 0x22, 0x4d, 0x18, 0x70, 0x50, 0x84}
|
||||
//var LZ4Header = []byte{0x04, 0x22, 0x4d, 0x18, 0x70, 0x50, 0x84}
|
||||
|
||||
// LZ4Footer - Footer of our LZ4 file
|
||||
var LZ4Footer = []byte{0x00, 0x00, 0x00, 0x00} // This is just an empty block
|
||||
|
||||
// Function that compresses a block using lz4
|
||||
func (c *Compression) compressBlockLz4(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize int64, err error) {
|
||||
// Write lz4 compressed data
|
||||
compressedBytes, err := lz4.Encode(nil, in)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
// Write compressed bytes
|
||||
n1, err := out.Write(compressedBytes)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
// Get checksum
|
||||
h := xxhash.New32()
|
||||
_, err = h.Write(compressedBytes[4:]) // The checksum doesn't include the size
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
checksum := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(checksum, h.Sum32())
|
||||
n2, err := out.Write(checksum)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
// Return sizes
|
||||
return uint32(n1 + n2), int64(len(in)), err
|
||||
const (
|
||||
frameMagic uint32 = 0x184D2204
|
||||
|
||||
compressedBlockFlag = 1 << 31
|
||||
compressedBlockMask = compressedBlockFlag - 1
|
||||
)
|
||||
|
||||
// AlgLz4 is the Lz4 Compression algorithm
|
||||
type AlgLz4 struct {
|
||||
Header lz4.Header
|
||||
buf [19]byte // magic number(4) + header(flags(2)+[Size(8)+DictID(4)]+checksum(1)) does not exceed 19 bytes
|
||||
}
|
||||
|
||||
// Utility function to decompress a block using LZ4
|
||||
func decompressBlockLz4(in io.Reader, out io.Writer, BlockSize int64) (n int, err error) {
|
||||
// InitializeLz4 creates an Lz4 compression algorithm
|
||||
func InitializeLz4(bs uint32, blockChecksum bool) Algorithm {
|
||||
a := new(AlgLz4)
|
||||
a.Header.Reset()
|
||||
a.Header = lz4.Header{
|
||||
BlockChecksum: blockChecksum,
|
||||
BlockMaxSize: int(bs),
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// GetFileExtension returns file extension
|
||||
func (a *AlgLz4) GetFileExtension() string {
|
||||
return ".lz4"
|
||||
}
|
||||
|
||||
// GetHeader returns the Lz4 compression header
|
||||
func (a *AlgLz4) GetHeader() []byte {
|
||||
// Size is optional.
|
||||
buf := a.buf[:]
|
||||
|
||||
// Set the fixed size data: magic number, block max size and flags.
|
||||
binary.LittleEndian.PutUint32(buf[0:], frameMagic)
|
||||
flg := byte(lz4.Version << 6)
|
||||
flg |= 1 << 5 // No block dependency.
|
||||
if a.Header.BlockChecksum {
|
||||
flg |= 1 << 4
|
||||
}
|
||||
if a.Header.Size > 0 {
|
||||
flg |= 1 << 3
|
||||
}
|
||||
buf[4] = flg
|
||||
buf[5] = blockSizeValueToIndex(a.Header.BlockMaxSize) << 4
|
||||
|
||||
// Current buffer size: magic(4) + flags(1) + block max size (1).
|
||||
n := 6
|
||||
if a.Header.Size > 0 {
|
||||
binary.LittleEndian.PutUint64(buf[n:], a.Header.Size)
|
||||
n += 8
|
||||
}
|
||||
|
||||
// The header checksum includes the flags, block max size and optional Size.
|
||||
buf[n] = byte(xxh32.ChecksumZero(buf[4:n]) >> 8 & 0xFF)
|
||||
|
||||
// Header ready, write it out.
|
||||
return buf[0 : n+1]
|
||||
}
|
||||
|
||||
// GetFooter returns
|
||||
func (a *AlgLz4) GetFooter() []byte {
|
||||
return LZ4Footer
|
||||
}
|
||||
|
||||
// CompressBlock that compresses a block using lz4
|
||||
func (a *AlgLz4) CompressBlock(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize uint64, err error) {
|
||||
if len(in) > 0 {
|
||||
n, err := a.compressBlock(in, out)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return n, uint64(len(in)), nil
|
||||
}
|
||||
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
// compressBlock compresses a block.
|
||||
func (a *AlgLz4) compressBlock(data []byte, dst io.Writer) (uint32, error) {
|
||||
zdata := make([]byte, a.Header.BlockMaxSize) // The compressed block size cannot exceed the input's.
|
||||
var zn int
|
||||
if level := a.Header.CompressionLevel; level != 0 {
|
||||
zn, _ = lz4.CompressBlockHC(data, zdata, level)
|
||||
} else {
|
||||
var hashTable [1 << 16]int
|
||||
zn, _ = lz4.CompressBlock(data, zdata, hashTable[:])
|
||||
}
|
||||
|
||||
var bLen uint32
|
||||
if zn > 0 && zn < len(data) {
|
||||
// Compressible and compressed size smaller than uncompressed: ok!
|
||||
bLen = uint32(zn)
|
||||
zdata = zdata[:zn]
|
||||
} else {
|
||||
// Uncompressed block.
|
||||
bLen = uint32(len(data)) | compressedBlockFlag
|
||||
zdata = data
|
||||
}
|
||||
|
||||
// Write the block.
|
||||
if err := a.writeUint32(bLen, dst); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err := dst.Write(zdata)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !a.Header.BlockChecksum {
|
||||
return bLen, nil
|
||||
}
|
||||
checksum := xxh32.ChecksumZero(zdata)
|
||||
if err := a.writeUint32(checksum, dst); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bLen, nil
|
||||
}
|
||||
|
||||
// writeUint32 writes a uint32 to the underlying writer.
|
||||
func (a *AlgLz4) writeUint32(x uint32, dst io.Writer) error {
|
||||
buf := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(buf, x)
|
||||
_, err := dst.Write(buf)
|
||||
return err
|
||||
}
|
||||
|
||||
func blockSizeValueToIndex(size int) byte {
|
||||
return 4 + byte(bits.TrailingZeros(uint(size)>>16)/2)
|
||||
}
|
||||
|
||||
// DecompressBlock decompresses Lz4 compressed block
|
||||
func (a *AlgLz4) DecompressBlock(in io.Reader, out io.Writer, BlockSize uint32) (n int, err error) {
|
||||
// Get our compressed data
|
||||
var b bytes.Buffer
|
||||
_, err = io.Copy(&b, in)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Add the length in byte form to the begining of the buffer. Because the length is not equal to BlockSize for the last block, the last block might screw this code up.
|
||||
compressedBytesWithHash := b.Bytes()
|
||||
compressedBytes := compressedBytesWithHash[:len(compressedBytesWithHash)-4]
|
||||
hash := compressedBytesWithHash[len(compressedBytesWithHash)-4:]
|
||||
// Verify, decode, write, and return
|
||||
h := xxhash.New32()
|
||||
_, err = h.Write(compressedBytes[4:])
|
||||
zdata := b.Bytes()
|
||||
bLen := binary.LittleEndian.Uint32(zdata[:4])
|
||||
|
||||
if bLen&compressedBlockFlag > 0 {
|
||||
// Uncompressed block.
|
||||
bLen &= compressedBlockMask
|
||||
|
||||
if bLen > BlockSize {
|
||||
return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
|
||||
}
|
||||
data := zdata[4 : bLen+4]
|
||||
|
||||
if a.Header.BlockChecksum {
|
||||
checksum := binary.LittleEndian.Uint32(zdata[4+bLen:])
|
||||
|
||||
if h := xxh32.ChecksumZero(data); h != checksum {
|
||||
return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
|
||||
}
|
||||
}
|
||||
_, err := out.Write(data)
|
||||
return len(data), err
|
||||
}
|
||||
|
||||
// compressed block
|
||||
if bLen > BlockSize {
|
||||
return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
|
||||
}
|
||||
|
||||
if a.Header.BlockChecksum {
|
||||
checksum := binary.LittleEndian.Uint32(zdata[4+bLen:])
|
||||
|
||||
if h := xxh32.ChecksumZero(zdata[4 : bLen+4]); h != checksum {
|
||||
return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
|
||||
}
|
||||
}
|
||||
|
||||
data := make([]byte, BlockSize)
|
||||
n, err = lz4.UncompressBlock(zdata[4:bLen+4], data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if binary.LittleEndian.Uint32(hash) != h.Sum32() {
|
||||
return 0, errors.New("XXHash checksum invalid")
|
||||
}
|
||||
dst := make([]byte, BlockSize*2)
|
||||
decompressed, err := lz4.Decode(dst, compressedBytes)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err = out.Write(decompressed)
|
||||
return len(decompressed), err
|
||||
_, err = out.Write(data[:n])
|
||||
return n, err
|
||||
}
|
||||
|
||||
@ -1,35 +0,0 @@
|
||||
package press
|
||||
|
||||
// This file implements compression/decompression using snappy.
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
// SnappyHeader - Header we add to a snappy file. We don't need this.
|
||||
var SnappyHeader = []byte{}
|
||||
|
||||
// Function that compresses a block using snappy
|
||||
func (c *Compression) compressBlockSnappy(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize int64, err error) {
|
||||
// Compress and return
|
||||
outBytes := snappy.Encode(nil, in)
|
||||
_, err = out.Write(outBytes)
|
||||
return uint32(len(outBytes)), int64(len(in)), err
|
||||
}
|
||||
|
||||
// Utility function to decompress a block using snappy
|
||||
func decompressBlockSnappy(in io.Reader, out io.Writer) (n int, err error) {
|
||||
var b bytes.Buffer
|
||||
_, err = io.Copy(&b, in)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
decompressed, err := snappy.Decode(nil, b.Bytes())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_, err = out.Write(decompressed)
|
||||
return len(decompressed), err
|
||||
}
|
||||
75
backend/press/alg_xz.go
Normal file
75
backend/press/alg_xz.go
Normal file
@ -0,0 +1,75 @@
|
||||
package press
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
// AlgXZ represents the XZ compression algorithm
|
||||
type AlgXZ struct {
|
||||
blockSize uint32
|
||||
config xz.WriterConfig
|
||||
}
|
||||
|
||||
// InitializeXZ creates an Lz4 compression algorithm
|
||||
func InitializeXZ(bs uint32) Algorithm {
|
||||
a := new(AlgXZ)
|
||||
a.blockSize = bs
|
||||
a.config = xz.WriterConfig{}
|
||||
return a
|
||||
}
|
||||
|
||||
// GetFileExtension returns file extension
|
||||
func (a *AlgXZ) GetFileExtension() string {
|
||||
return ".xz"
|
||||
}
|
||||
|
||||
// GetHeader returns the Lz4 compression header
|
||||
func (a *AlgXZ) GetHeader() []byte {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
// GetFooter returns
|
||||
func (a *AlgXZ) GetFooter() []byte {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
// CompressBlock that compresses a block using lz4
|
||||
func (a *AlgXZ) CompressBlock(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize uint64, err error) {
|
||||
// Initialize buffer
|
||||
bufw := bufio.NewWriterSize(out, int(a.blockSize+(a.blockSize)>>4))
|
||||
|
||||
// Initialize block writer
|
||||
outw, err := a.config.NewWriter(bufw)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Compress block
|
||||
_, err = outw.Write(in)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
// Finalize gzip file, flush buffer and return
|
||||
err = outw.Close()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
blockSize := uint32(bufw.Buffered())
|
||||
err = bufw.Flush()
|
||||
|
||||
return blockSize, uint64(len(in)), err
|
||||
}
|
||||
|
||||
// DecompressBlock decompresses Lz4 compressed block
|
||||
func (a *AlgXZ) DecompressBlock(in io.Reader, out io.Writer, BlockSize uint32) (n int, err error) {
|
||||
xzReader, err := xz.NewReader(in)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written, err := io.Copy(out, xzReader)
|
||||
return int(written), err
|
||||
}
|
||||
@ -2,26 +2,11 @@
|
||||
// This file is the backend implementation for seekable compression.
|
||||
package press
|
||||
|
||||
/*
|
||||
NOTES:
|
||||
Structure of the metadata we store is:
|
||||
gzipExtraify(gzip([4-byte header size][4-byte block size] ... [4-byte block size][4-byte raw size of last block]))
|
||||
This is appended to any compressed file, and is ignored as trailing garbage in our LZ4 and SNAPPY implementations, and seen as empty archives in our GZIP and XZ_IN_GZ implementations.
|
||||
|
||||
There are two possible compression/decompression function pairs to be used:
|
||||
The two functions that store data internally are:
|
||||
- Compression.CompressFileAppendingBlockData. Appends block data in extra data fields of empty gzip files at the end.
|
||||
- DecompressFile. Reads block data from extra fields of these empty gzip files.
|
||||
The two functions that require externally stored data are:
|
||||
- Compression.CompressFileReturningBlockData. Returns a []uint32 containing raw (uncompressed and unencoded) block data, which must be externally stored.
|
||||
- DecompressFileExtData. Takes in the []uint32 that was returned by Compression.CompressFileReturningBlockData
|
||||
WARNING: These function pairs are incompatible with each other. Don't use CompressFileAppendingBlockData with DecompressFileExtData, or the other way around. It won't work.
|
||||
*/
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
@ -30,14 +15,9 @@ import (
|
||||
// Compression modes
|
||||
const (
|
||||
Uncompressed = -1
|
||||
GzipStore = 0
|
||||
GzipMin = 1
|
||||
GzipDefault = 2
|
||||
GzipMax = 3
|
||||
LZ4 = 4
|
||||
Snappy = 5
|
||||
XZMin = 6
|
||||
XZDefault = 7
|
||||
LZ4 = 2
|
||||
Gzip = 4
|
||||
XZ = 8
|
||||
)
|
||||
|
||||
// Errors
|
||||
@ -50,7 +30,8 @@ const DEBUG = false
|
||||
|
||||
// Compression is a struct containing configurable variables (what used to be constants)
|
||||
type Compression struct {
|
||||
CompressionMode int // Compression mode
|
||||
CompressionMode int // Compression mode
|
||||
Algorithm Algorithm
|
||||
BlockSize uint32 // Size of blocks. Higher block size means better compression but more download bandwidth needed for small downloads
|
||||
// ~1MB is recommended for xz, while ~128KB is recommended for gzip and lz4
|
||||
HeuristicBytes int64 // Bytes to perform gzip heuristic on to determine whether a file should be compressed
|
||||
@ -59,23 +40,31 @@ type Compression struct {
|
||||
BinPath string // Path to compression binary. This is used for all non-gzip compression.
|
||||
}
|
||||
|
||||
// Algorithm is the main compression Algorithm Interface
|
||||
type Algorithm interface {
|
||||
GetHeader() []byte
|
||||
|
||||
GetFileExtension() string
|
||||
|
||||
CompressBlock(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize uint64, err error)
|
||||
|
||||
DecompressBlock(in io.Reader, out io.Writer, BlockSize uint32) (n int, err error)
|
||||
|
||||
GetFooter() []byte
|
||||
}
|
||||
|
||||
// NewCompressionPreset creates a Compression object with a preset mode/bs
|
||||
func NewCompressionPreset(preset string) (*Compression, error) {
|
||||
switch preset {
|
||||
case "gzip-store":
|
||||
return NewCompression(GzipStore, 131070) // GZIP-store (dummy) compression
|
||||
case "lz4":
|
||||
return NewCompression(LZ4, 262140) // LZ4 compression (very fast)
|
||||
case "snappy":
|
||||
return NewCompression(Snappy, 262140) // Snappy compression (like LZ4, but slower and worse)
|
||||
case "gzip-min":
|
||||
return NewCompression(GzipMin, 131070) // GZIP-min compression (fast)
|
||||
case "gzip-default":
|
||||
return NewCompression(GzipDefault, 131070) // GZIP-default compression (medium)
|
||||
case "xz-min":
|
||||
return NewCompression(XZMin, 524288) // XZ-min compression (slow)
|
||||
case "xz-default":
|
||||
return NewCompression(XZDefault, 1048576) // XZ-default compression (very slow)
|
||||
alg := InitializeLz4(262144, true)
|
||||
return NewCompression(LZ4, alg, 262144) // LZ4 compression (very fast)
|
||||
case "gzip":
|
||||
alg := InitializeGzip(131072, 6)
|
||||
return NewCompression(Gzip, alg, 131070) // GZIP-default compression (medium)*/
|
||||
case "xz":
|
||||
alg := InitializeXZ(1048576)
|
||||
return NewCompression(XZ, alg, 1048576) // XZ compression (strong compression)*/
|
||||
}
|
||||
return nil, errors.New("Compression mode doesn't exist")
|
||||
}
|
||||
@ -83,66 +72,45 @@ func NewCompressionPreset(preset string) (*Compression, error) {
|
||||
// NewCompressionPresetNumber creates a Compression object with a preset mode/bs
|
||||
func NewCompressionPresetNumber(preset int) (*Compression, error) {
|
||||
switch preset {
|
||||
case GzipStore:
|
||||
return NewCompression(GzipStore, 131070) // GZIP-store (dummy) compression
|
||||
case LZ4:
|
||||
return NewCompression(LZ4, 262140) // LZ4 compression (very fast)
|
||||
case Snappy:
|
||||
return NewCompression(Snappy, 262140) // Snappy compression (like LZ4, but slower and worse)
|
||||
case GzipMin:
|
||||
return NewCompression(GzipMin, 131070) // GZIP-min compression (fast)
|
||||
case GzipDefault:
|
||||
return NewCompression(GzipDefault, 131070) // GZIP-default compression (medium)
|
||||
case XZMin:
|
||||
return NewCompression(XZMin, 524288) // XZ-min compression (slow)
|
||||
case XZDefault:
|
||||
return NewCompression(XZDefault, 1048576) // XZ-default compression (very slow)
|
||||
alg := InitializeLz4(262144, true)
|
||||
return NewCompression(LZ4, alg, 262144) // LZ4 compression (very fast)
|
||||
case Gzip:
|
||||
alg := InitializeGzip(131072, 6)
|
||||
return NewCompression(Gzip, alg, 131070) // GZIP-default compression (medium)*/
|
||||
case XZ:
|
||||
alg := InitializeXZ(1048576)
|
||||
return NewCompression(XZ, alg, 1048576) // XZ compression (strong compression)*/
|
||||
}
|
||||
return nil, errors.New("Compression mode doesn't exist")
|
||||
}
|
||||
|
||||
// NewCompression creates a Compression object with some default configuration values
|
||||
func NewCompression(mode int, bs uint32) (*Compression, error) {
|
||||
return NewCompressionAdvanced(mode, bs, 1048576, 12, 0.9)
|
||||
func NewCompression(mode int, alg Algorithm, bs uint32) (*Compression, error) {
|
||||
return NewCompressionAdvanced(mode, alg, bs, 1048576, 12, 0.9)
|
||||
}
|
||||
|
||||
// NewCompressionAdvanced creates a Compression object
|
||||
func NewCompressionAdvanced(mode int, bs uint32, hb int64, threads int, mcr float64) (c *Compression, err error) {
|
||||
func NewCompressionAdvanced(mode int, alg Algorithm, bs uint32, hb int64, threads int, mcr float64) (c *Compression, err error) {
|
||||
// Set vars
|
||||
c = new(Compression)
|
||||
c.Algorithm = alg
|
||||
c.CompressionMode = mode
|
||||
c.BlockSize = bs
|
||||
c.HeuristicBytes = hb
|
||||
c.NumThreads = threads
|
||||
c.MaxCompressionRatio = mcr
|
||||
// Get binary path if needed
|
||||
err = getBinPaths(c, mode)
|
||||
return c, err
|
||||
}
|
||||
|
||||
/*** UTILITY FUNCTIONS ***/
|
||||
// Gets an overestimate for the maximum compressed block size
|
||||
func (c *Compression) maxCompressedBlockSize() uint32 {
|
||||
return c.BlockSize + (c.BlockSize >> 2) + 256
|
||||
}
|
||||
|
||||
// GetFileExtension gets a file extension for current compression mode
|
||||
func (c *Compression) GetFileExtension() string {
|
||||
switch c.CompressionMode {
|
||||
case GzipStore, GzipMin, GzipDefault, GzipMax:
|
||||
return ".gz"
|
||||
case XZMin, XZDefault:
|
||||
return ".xzgz"
|
||||
case LZ4:
|
||||
return ".lz4"
|
||||
case Snappy:
|
||||
return ".snap"
|
||||
}
|
||||
panic("Compression mode doesn't exist")
|
||||
return c.Algorithm.GetFileExtension()
|
||||
}
|
||||
|
||||
// GetFileCompressionInfo gets a file extension along with compressibility of file
|
||||
// It is currently not being used but may be usable in the future.
|
||||
func (c *Compression) GetFileCompressionInfo(reader io.Reader) (compressable bool, extension string, err error) {
|
||||
// Use our compression algorithm to do a heuristic on the first few bytes
|
||||
var emulatedBlock, emulatedBlockCompressed bytes.Buffer
|
||||
@ -150,7 +118,7 @@ func (c *Compression) GetFileCompressionInfo(reader io.Reader) (compressable boo
|
||||
if err != nil && err != io.EOF {
|
||||
return false, "", err
|
||||
}
|
||||
compressedSize, uncompressedSize, err := c.compressBlock(emulatedBlock.Bytes(), &emulatedBlockCompressed)
|
||||
compressedSize, uncompressedSize, err := c.Algorithm.CompressBlock(emulatedBlock.Bytes(), &emulatedBlockCompressed)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
@ -162,79 +130,25 @@ func (c *Compression) GetFileCompressionInfo(reader io.Reader) (compressable boo
|
||||
}
|
||||
|
||||
// If the file is compressible, select file extension based on compression mode
|
||||
return true, c.GetFileExtension(), nil
|
||||
}
|
||||
|
||||
// Gets the file header we add to files of the currently used algorithm. Currently only used for lz4.
|
||||
func (c *Compression) getHeader() []byte {
|
||||
switch c.CompressionMode {
|
||||
case GzipStore, GzipMin, GzipDefault, GzipMax:
|
||||
return GzipHeader
|
||||
case XZMin, XZDefault:
|
||||
return ExecHeader
|
||||
case LZ4:
|
||||
return LZ4Header
|
||||
case Snappy:
|
||||
return SnappyHeader
|
||||
}
|
||||
panic("Compression mode doesn't exist")
|
||||
}
|
||||
|
||||
// Gets the file footer we add to files of the currently used algorithm. Currently only used for lz4.
|
||||
func (c *Compression) getFooter() []byte {
|
||||
switch c.CompressionMode {
|
||||
case GzipStore, GzipMin, GzipDefault, GzipMax:
|
||||
return []byte{}
|
||||
case XZMin, XZDefault:
|
||||
return []byte{}
|
||||
case LZ4:
|
||||
return LZ4Footer
|
||||
case Snappy:
|
||||
return []byte{}
|
||||
}
|
||||
panic("Compression mode doesn't exist")
|
||||
}
|
||||
|
||||
/*** BLOCK COMPRESSION FUNCTIONS ***/
|
||||
// Wrapper function to compress a block
|
||||
func (c *Compression) compressBlock(in []byte, out io.Writer) (compressedSize uint32, uncompressedSize int64, err error) {
|
||||
switch c.CompressionMode { // Select compression function (and arguments) based on compression mode
|
||||
case GzipStore:
|
||||
return c.compressBlockGz(in, out, 0)
|
||||
case GzipMin:
|
||||
return c.compressBlockGz(in, out, 1)
|
||||
case GzipDefault:
|
||||
return c.compressBlockGz(in, out, 6)
|
||||
case GzipMax:
|
||||
return c.compressBlockGz(in, out, 9)
|
||||
case XZDefault:
|
||||
return c.compressBlockExec(in, out, c.BinPath, []string{"-c"})
|
||||
case XZMin:
|
||||
return c.compressBlockExec(in, out, c.BinPath, []string{"-c1"})
|
||||
case LZ4:
|
||||
return c.compressBlockLz4(in, out)
|
||||
case Snappy:
|
||||
return c.compressBlockSnappy(in, out)
|
||||
}
|
||||
panic("Compression mode doesn't exist")
|
||||
return true, c.Algorithm.GetFileExtension(), nil
|
||||
}
|
||||
|
||||
/*** MAIN COMPRESSION INTERFACE ***/
|
||||
// compressionResult represents the result of compression for a single block (gotten by a single thread)
|
||||
type compressionResult struct {
|
||||
buffer *bytes.Buffer
|
||||
n int64
|
||||
n uint64
|
||||
err error
|
||||
}
|
||||
|
||||
// CompressFileReturningBlockData compresses a file returning the block data for that file.
|
||||
func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer) (blockData []uint32, err error) {
|
||||
// Initialize buffered writer
|
||||
bufw := bufio.NewWriterSize(out, int(c.maxCompressedBlockSize()*uint32(c.NumThreads)))
|
||||
bufw := bufio.NewWriterSize(out, int((c.BlockSize+(c.BlockSize)>>4)*uint32(c.NumThreads)))
|
||||
|
||||
// Get blockData, copy over header, add length of header to blockData
|
||||
blockData = make([]uint32, 0)
|
||||
header := c.getHeader()
|
||||
header := c.Algorithm.GetHeader()
|
||||
_, err = bufw.Write(header)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -263,7 +177,7 @@ func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer
|
||||
var buffer bytes.Buffer
|
||||
|
||||
// Compress block
|
||||
_, n, err := c.compressBlock(in, &buffer)
|
||||
_, n, err := c.Algorithm.CompressBlock(in, &buffer)
|
||||
if err != nil && err != io.EOF { // This errored out.
|
||||
res.buffer = nil
|
||||
res.n = 0
|
||||
@ -276,7 +190,6 @@ func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer
|
||||
res.n = n
|
||||
res.err = err
|
||||
compressionResults[i] <- res
|
||||
return
|
||||
}(i, inputBuffer.Bytes())
|
||||
// If we have reached eof, we don't need more threads
|
||||
if eofAt != -1 {
|
||||
@ -294,12 +207,13 @@ func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer
|
||||
return nil, res.err
|
||||
}
|
||||
blockSize := uint32(res.buffer.Len())
|
||||
|
||||
_, err = io.Copy(bufw, res.buffer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if DEBUG {
|
||||
log.Printf("%d %d\n", res.n, blockSize)
|
||||
fmt.Printf("%d %d\n", res.n, blockSize)
|
||||
}
|
||||
|
||||
// Append block size to block data
|
||||
@ -310,6 +224,7 @@ func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer
|
||||
if DEBUG {
|
||||
log.Printf("%d %d %d\n", res.n, byte(res.n%256), byte(res.n/256))
|
||||
}
|
||||
|
||||
blockData = append(blockData, uint32(res.n))
|
||||
break
|
||||
}
|
||||
@ -333,7 +248,9 @@ func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer
|
||||
}
|
||||
|
||||
// Write footer and flush
|
||||
_, err = bufw.Write(c.getFooter())
|
||||
footer := c.Algorithm.GetFooter()
|
||||
|
||||
_, err = bufw.Write(footer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -344,22 +261,6 @@ func (c *Compression) CompressFileReturningBlockData(in io.Reader, out io.Writer
|
||||
}
|
||||
|
||||
/*** BLOCK DECOMPRESSION FUNCTIONS ***/
|
||||
// Wrapper function to decompress a block
|
||||
func (d *Decompressor) decompressBlock(in io.Reader, out io.Writer) (n int, err error) {
|
||||
switch d.c.CompressionMode { // Select decompression function based off compression mode
|
||||
case GzipStore, GzipMin, GzipDefault, GzipMax:
|
||||
return decompressBlockRangeGz(in, out)
|
||||
case XZMin:
|
||||
return decompressBlockRangeExec(in, out, d.c.BinPath, []string{"-dc1"})
|
||||
case XZDefault:
|
||||
return decompressBlockRangeExec(in, out, d.c.BinPath, []string{"-dc"})
|
||||
case LZ4:
|
||||
return decompressBlockLz4(in, out, int64(d.c.BlockSize))
|
||||
case Snappy:
|
||||
return decompressBlockSnappy(in, out)
|
||||
}
|
||||
panic("Compression mode doesn't exist") // If none of the above returned
|
||||
}
|
||||
|
||||
// Wrapper function for decompressBlock that implements multithreading
|
||||
// decompressionResult represents the result of decompressing a block
|
||||
@ -412,7 +313,7 @@ func (d *Decompressor) decompressBlockRangeMultithreaded(in io.Reader, out io.Wr
|
||||
var res decompressionResult
|
||||
|
||||
// Decompress block
|
||||
_, res.err = d.decompressBlock(in, &block)
|
||||
_, res.err = d.c.Algorithm.DecompressBlock(in, &block, d.c.BlockSize)
|
||||
res.buffer = &block
|
||||
decompressionResults[i] <- res
|
||||
}(i, currBlock, &compressedBlock)
|
||||
@ -535,8 +436,7 @@ func (d Decompressor) Read(p []byte) (int, error) {
|
||||
blocksToRead = int64(d.numBlocks) - blockNumber
|
||||
returnEOF = true
|
||||
}
|
||||
var blockEnd int64 // End position of blocks to read
|
||||
blockEnd = d.blockStarts[blockNumber+blocksToRead] // Start of the block after the last block we want to get is the end of the last block we want to get
|
||||
blockEnd := d.blockStarts[blockNumber+blocksToRead] // Start of the block after the last block we want to get is the end of the last block we want to get
|
||||
blockLen := blockEnd - blockStart
|
||||
|
||||
// Read compressed block range into buffer
|
||||
|
||||
@ -122,12 +122,7 @@ func getCompressibleString(size int) string {
|
||||
}
|
||||
|
||||
func TestCompression(t *testing.T) {
|
||||
testCases := []string{"lz4", "snappy", "gzip-min"}
|
||||
if checkXZ() {
|
||||
testCases = append(testCases, "xz-min")
|
||||
} else {
|
||||
t.Log("XZ binary not found on current system. Not testing xz.")
|
||||
}
|
||||
testCases := []string{"lz4", "gzip", "xz"}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc, func(t *testing.T) {
|
||||
testSmallLarge(t, tc)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -6,9 +6,9 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
_ "github.com/ncw/rclone/backend/local"
|
||||
"github.com/ncw/rclone/fstest"
|
||||
"github.com/ncw/rclone/fstest/fstests"
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/fstest/fstests"
|
||||
)
|
||||
|
||||
// TestIntegration runs integration tests against the remote
|
||||
@ -17,10 +17,21 @@ func TestIntegration(t *testing.T) {
|
||||
t.Skip("Skipping as -remote not set")
|
||||
}
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: *fstest.RemoteName,
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{"OpenWriterAt"},
|
||||
UnimplementableObjectMethods: []string{},
|
||||
RemoteName: *fstest.RemoteName,
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{
|
||||
"OpenWriterAt",
|
||||
"MergeDirs",
|
||||
"DirCacheFlush",
|
||||
"PutUnchecked",
|
||||
"PutStream",
|
||||
"UserInfo",
|
||||
"Disconnect",
|
||||
},
|
||||
UnimplementableObjectMethods: []string{
|
||||
"GetTier",
|
||||
"SetTier",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@ -32,10 +43,21 @@ func TestRemoteLz4(t *testing.T) {
|
||||
tempdir := filepath.Join(os.TempDir(), "rclone-press-test-lz4")
|
||||
name := "TestPressLz4"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{"OpenWriterAt"},
|
||||
UnimplementableObjectMethods: []string{},
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{
|
||||
"OpenWriterAt",
|
||||
"MergeDirs",
|
||||
"DirCacheFlush",
|
||||
"PutUnchecked",
|
||||
"PutStream",
|
||||
"UserInfo",
|
||||
"Disconnect",
|
||||
},
|
||||
UnimplementableObjectMethods: []string{
|
||||
"GetTier",
|
||||
"SetTier",
|
||||
},
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "press"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
@ -52,37 +74,56 @@ func TestRemoteGzip(t *testing.T) {
|
||||
tempdir := filepath.Join(os.TempDir(), "rclone-press-test-gzip")
|
||||
name := "TestPressGzip"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{"OpenWriterAt"},
|
||||
UnimplementableObjectMethods: []string{},
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{
|
||||
"OpenWriterAt",
|
||||
"MergeDirs",
|
||||
"DirCacheFlush",
|
||||
"PutUnchecked",
|
||||
"PutStream",
|
||||
"UserInfo",
|
||||
"Disconnect",
|
||||
},
|
||||
UnimplementableObjectMethods: []string{
|
||||
"GetTier",
|
||||
"SetTier",
|
||||
},
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "press"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
{Name: name, Key: "compression_mode", Value: "gzip-min"},
|
||||
{Name: name, Key: "compression_mode", Value: "gzip"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestRemoteXZ tests XZ compression
|
||||
func TestRemoteXZ(t *testing.T) {
|
||||
if !checkXZ() {
|
||||
t.Skip("XZ binary not found on current system")
|
||||
}
|
||||
// TestRemoteXz tests XZ compression
|
||||
func TestRemoteXz(t *testing.T) {
|
||||
if *fstest.RemoteName != "" {
|
||||
t.Skip("Skipping as -remote set")
|
||||
}
|
||||
tempdir := filepath.Join(os.TempDir(), "rclone-press-test-xz")
|
||||
name := "TestPressXZ"
|
||||
name := "TestPressXz"
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{"OpenWriterAt"},
|
||||
UnimplementableObjectMethods: []string{},
|
||||
RemoteName: name + ":",
|
||||
NilObject: (*Object)(nil),
|
||||
UnimplementableFsMethods: []string{
|
||||
"OpenWriterAt",
|
||||
"MergeDirs",
|
||||
"DirCacheFlush",
|
||||
"PutUnchecked",
|
||||
"PutStream",
|
||||
"UserInfo",
|
||||
"Disconnect",
|
||||
},
|
||||
UnimplementableObjectMethods: []string{
|
||||
"GetTier",
|
||||
"SetTier",
|
||||
},
|
||||
ExtraConfig: []fstests.ExtraConfigItem{
|
||||
{Name: name, Key: "type", Value: "press"},
|
||||
{Name: name, Key: "remote", Value: tempdir},
|
||||
{Name: name, Key: "compression_mode", Value: "xz-min"},
|
||||
{Name: name, Key: "compression_mode", Value: "xz"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
5
go.mod
5
go.mod
@ -8,7 +8,6 @@ require (
|
||||
github.com/Azure/azure-pipeline-go v0.2.2
|
||||
github.com/Azure/azure-storage-blob-go v0.10.0
|
||||
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
|
||||
github.com/OneOfOne/xxhash v1.2.7
|
||||
github.com/a8m/tree v0.0.0-20181222104329-6a0b80129de4
|
||||
github.com/aalpar/deheap v0.0.0-20200318053559-9a0c2883bd56
|
||||
github.com/abbot/go-http-auth v0.4.0
|
||||
@ -17,9 +16,11 @@ require (
|
||||
github.com/aws/aws-sdk-go v1.32.11
|
||||
github.com/billziss-gh/cgofuse v1.4.0
|
||||
github.com/btcsuite/btcutil v1.0.2 // indirect
|
||||
github.com/buengese/xxh32 v1.0.1
|
||||
github.com/calebcase/tmpfile v1.0.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial v5.6.0+incompatible
|
||||
github.com/gabriel-vasile/mimetype v1.1.1
|
||||
github.com/gogo/protobuf v1.3.1 // indirect
|
||||
github.com/google/go-querystring v1.0.0 // indirect
|
||||
github.com/hanwen/go-fuse/v2 v2.0.3
|
||||
@ -38,6 +39,7 @@ require (
|
||||
github.com/nsf/termbox-go v0.0.0-20200418040025-38ba6e5628f1
|
||||
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pierrec/lz4 v2.4.1+incompatible
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/pkg/sftp v1.11.0
|
||||
github.com/prometheus/client_golang v1.7.1
|
||||
@ -50,6 +52,7 @@ require (
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20200416171014-ffad7fcb44b8
|
||||
github.com/ulikunitz/xz v0.5.8
|
||||
github.com/xanzy/ssh-agent v0.2.1
|
||||
github.com/youmark/pkcs8 v0.0.0-20200520070018-fad002e585ce
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0
|
||||
|
||||
20
go.sum
20
go.sum
@ -56,8 +56,7 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/OneOfOne/xxhash v1.2.7 h1:fzrmmkskv067ZQbd9wERNGuxckWw67dyzoMG62p7LMo=
|
||||
github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w=
|
||||
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
|
||||
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
|
||||
@ -104,6 +103,7 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/buengese/xxh32 v1.0.1/go.mod h1:Q5GTtu7m/GuqzCc8YZ0n+oetaGFwW7oy291HvqLTZFk=
|
||||
github.com/calebcase/tmpfile v1.0.2-0.20200602150926-3af473ef8439/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
|
||||
github.com/calebcase/tmpfile v1.0.2 h1:1AGuhKiUu4J6wxz6lxuF6ck3f8G2kaV6KSEny0RGCig=
|
||||
github.com/calebcase/tmpfile v1.0.2/go.mod h1:iErLeG/iqJr8LaQ/gYRv4GXdqssi3jg4iSzvrA06/lw=
|
||||
@ -142,11 +142,15 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/frankban/quicktest v1.7.3 h1:kV0lw0TH1j1hozahVmcpFCsbV5hcS4ZalH+U7UoeTow=
|
||||
github.com/frankban/quicktest v1.7.3/go.mod h1:V1d2J5pfxYH6EjBAgSK7YNXcXlTWxUHdE1sVDXkjnig=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gabriel-vasile/mimetype v1.0.2 h1:GKCo1TUCg0pV0R4atTcaLv/9SI2W9xPgMySZxUxcJOE=
|
||||
github.com/gabriel-vasile/mimetype v1.0.2/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To=
|
||||
github.com/gabriel-vasile/mimetype v1.1.1 h1:qbN9MPuRf3bstHu9zkI9jDWNfH//9+9kHxr9oRBBBOA=
|
||||
github.com/gabriel-vasile/mimetype v1.1.1/go.mod h1:6CDPel/o/3/s4+bp6kIbsWATq8pmgOisOPG40CJa6To=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
@ -235,8 +239,6 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/id01/go-lz4 v1.0.3 h1:D3krbAf5BppFsRSVa75yFo+JMxlTqFwuYpyHQAOgYds=
|
||||
github.com/id01/go-lz4 v1.0.3/go.mod h1:G8scWkW5nw6fEwIREHZcWy3qddP/Go9IImmcit+bTzw=
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
@ -335,6 +337,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 h1:XeOYlK9W1uCmhjJSsY78Mcuh7MVkNjTzmHx1yBzizSU=
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14/go.mod h1:jVblp62SafmidSkvWrXyxAme3gaTfEtWwRPGz5cpvHg=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg=
|
||||
github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@ -421,6 +425,10 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
|
||||
github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ=
|
||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
|
||||
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
@ -759,6 +767,8 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user