zrepl/util/chunking/chunking.go

package chunking

import (
	"bytes"
	"encoding/binary"
	"io"
)

var ChunkBufSize uint32 = 32 * 1024
var ChunkHeaderByteOrder = binary.LittleEndian

type Unchunker struct {
	ChunkCount          int
	in                  io.Reader
	remainingChunkBytes uint32
	finishErr           error
}

func NewUnchunker(conn io.Reader) *Unchunker {
	return &Unchunker{
		in:                  conn,
		remainingChunkBytes: 0,
	}
}

func (c *Unchunker) Read(b []byte) (n int, err error) {

	if c.finishErr != nil {
		return 0, c.finishErr
	}

	if c.remainingChunkBytes == 0 {

		var nextChunkLen uint32
		err = binary.Read(c.in, ChunkHeaderByteOrder, &nextChunkLen)
		if err != nil {
			c.finishErr = err // can't handle this
			return
		}

		// A chunk of len 0 indicates end of stream
		if nextChunkLen == 0 {
			c.finishErr = io.EOF
			return 0, c.finishErr
		}

		c.remainingChunkBytes = nextChunkLen
		c.ChunkCount++

	}

	if c.remainingChunkBytes == 0 {
		panic("internal inconsistency: c.remainingChunkBytes must be > 0")
	}
	if len(b) <= 0 {
		panic("cannot read into buffer of length 0")
	}

	maxRead := min(int(c.remainingChunkBytes), len(b))

	n, err = c.in.Read(b[0:maxRead])
	if err != nil {
		return
	}
	c.remainingChunkBytes -= uint32(n)

	return

}

func (c *Unchunker) Close() (err error) {

	buf := make([]byte, 4096)
	for err == nil {
		_, err = c.Read(buf)
		if err == io.EOF {
			err = nil
			break
		}
	}
	return
}

func min(a, b int) int {
	if a < b {
		return a
	} else {
		return b
	}
}

type Chunker struct {
	ChunkCount          int
	in                  io.Reader
	inEOF               bool
	remainingChunkBytes int
	payloadBufLen       int
	payloadBuf          []byte
	headerBuf           *bytes.Buffer
	finalHeaderBuffered bool
}

func NewChunker(conn io.Reader) Chunker {
	return NewChunkerSized(conn, ChunkBufSize)
}

func NewChunkerSized(conn io.Reader, chunkSize uint32) Chunker {

	buf := make([]byte, int(chunkSize)-binary.Size(chunkSize))

	return Chunker{
		in:                  conn,
		remainingChunkBytes: 0,
		payloadBufLen:       0,
		payloadBuf:          buf,
		headerBuf:           &bytes.Buffer{},
	}

}

func (c *Chunker) Read(b []byte) (n int, err error) {

	if len(b) == 0 {
		panic("unexpected empty output buffer")
	}

	if c.inEOF && c.finalHeaderBuffered && c.headerBuf.Len() == 0 { // all bufs empty and no more bytes to expect
		return 0, io.EOF
	}

	n = 0

	if c.headerBuf.Len() > 0 { // first drain the header buf
		nh, err := c.headerBuf.Read(b[n:])
		if nh > 0 {
			n += nh
		}
		if nh == 0 || (err != nil && err != io.EOF) {
			panic("unexpected behavior: in-memory buffer should not throw errors")
		}
		if c.headerBuf.Len() != 0 {
			return n, nil // finish writing the header before we can proceed with payload
		}
		if c.finalHeaderBuffered {
			// we just wrote the final header
			return n, io.EOF
		}
	}

	if c.remainingChunkBytes > 0 { // then drain the payload buf

		npl := copy(b[n:], c.payloadBuf[c.payloadBufLen-c.remainingChunkBytes:c.payloadBufLen])
		c.remainingChunkBytes -= npl
		if c.remainingChunkBytes < 0 {
			panic("unexpected behavior, copy() should not copy more than max(cap(), len())")
		}
		n += npl
	}

	if c.remainingChunkBytes == 0 && !c.inEOF { // fillup bufs

		newPayloadLen, err := c.in.Read(c.payloadBuf)

		if newPayloadLen > 0 {
			c.payloadBufLen = newPayloadLen
			c.remainingChunkBytes = newPayloadLen
		}
		if err == io.EOF {
			c.inEOF = true
		} else if err != nil {
			return n, err
		}
		if newPayloadLen == 0 { // apparently, this happens with some Readers
			c.finalHeaderBuffered = true
		}

		// Fill header buf
		{
			c.headerBuf.Reset()
			nextChunkLen := uint32(newPayloadLen)
			err := binary.Write(c.headerBuf, ChunkHeaderByteOrder, nextChunkLen)
			if err != nil {
				panic("unexpected error, write to in-memory buffer should not throw error")
			}
		}

		if c.headerBuf.Len() == 0 {
			panic("unexpected empty header buf")
		}

	} else if c.remainingChunkBytes == 0 && c.inEOF && !c.finalHeaderBuffered {

		c.headerBuf.Reset()
		err := binary.Write(c.headerBuf, ChunkHeaderByteOrder, uint32(0))
		if err != nil {
			panic("unexpected error, write to in-memory buffer should not throw error [2]")
		}
		c.finalHeaderBuffered = true

	}

	return

}