mirror of
https://github.com/rclone/rclone.git
synced 2024-12-05 14:50:57 +01:00
491 lines
14 KiB
Go
491 lines
14 KiB
Go
// Copyright 2014 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// Package hpack implements HPACK, a compression format for
|
|
// efficiently representing HTTP header fields in the context of HTTP/2.
|
|
//
|
|
// See http://tools.ietf.org/html/draft-ietf-httpbis-header-compression-09
|
|
package hpack
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
)
|
|
|
|
// A DecodingError is something the spec defines as a decoding error.
|
|
type DecodingError struct {
|
|
Err error
|
|
}
|
|
|
|
func (de DecodingError) Error() string {
|
|
return fmt.Sprintf("decoding error: %v", de.Err)
|
|
}
|
|
|
|
// An InvalidIndexError is returned when an encoder references a table
|
|
// entry before the static table or after the end of the dynamic table.
|
|
type InvalidIndexError int
|
|
|
|
func (e InvalidIndexError) Error() string {
|
|
return fmt.Sprintf("invalid indexed representation index %d", int(e))
|
|
}
|
|
|
|
// A HeaderField is a name-value pair. Both the name and value are
|
|
// treated as opaque sequences of octets.
|
|
type HeaderField struct {
|
|
Name, Value string
|
|
|
|
// Sensitive means that this header field should never be
|
|
// indexed.
|
|
Sensitive bool
|
|
}
|
|
|
|
// IsPseudo reports whether the header field is an http2 pseudo header.
|
|
// That is, it reports whether it starts with a colon.
|
|
// It is not otherwise guaranteed to be a valid pseudo header field,
|
|
// though.
|
|
func (hf HeaderField) IsPseudo() bool {
|
|
return len(hf.Name) != 0 && hf.Name[0] == ':'
|
|
}
|
|
|
|
func (hf HeaderField) String() string {
|
|
var suffix string
|
|
if hf.Sensitive {
|
|
suffix = " (sensitive)"
|
|
}
|
|
return fmt.Sprintf("header field %q = %q%s", hf.Name, hf.Value, suffix)
|
|
}
|
|
|
|
// Size returns the size of an entry per RFC 7541 section 4.1.
|
|
func (hf HeaderField) Size() uint32 {
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.4.1
|
|
// "The size of the dynamic table is the sum of the size of
|
|
// its entries. The size of an entry is the sum of its name's
|
|
// length in octets (as defined in Section 5.2), its value's
|
|
// length in octets (see Section 5.2), plus 32. The size of
|
|
// an entry is calculated using the length of the name and
|
|
// value without any Huffman encoding applied."
|
|
|
|
// This can overflow if somebody makes a large HeaderField
|
|
// Name and/or Value by hand, but we don't care, because that
|
|
// won't happen on the wire because the encoding doesn't allow
|
|
// it.
|
|
return uint32(len(hf.Name) + len(hf.Value) + 32)
|
|
}
|
|
|
|
// A Decoder is the decoding context for incremental processing of
|
|
// header blocks.
|
|
type Decoder struct {
|
|
dynTab dynamicTable
|
|
emit func(f HeaderField)
|
|
|
|
emitEnabled bool // whether calls to emit are enabled
|
|
maxStrLen int // 0 means unlimited
|
|
|
|
// buf is the unparsed buffer. It's only written to
|
|
// saveBuf if it was truncated in the middle of a header
|
|
// block. Because it's usually not owned, we can only
|
|
// process it under Write.
|
|
buf []byte // not owned; only valid during Write
|
|
|
|
// saveBuf is previous data passed to Write which we weren't able
|
|
// to fully parse before. Unlike buf, we own this data.
|
|
saveBuf bytes.Buffer
|
|
}
|
|
|
|
// NewDecoder returns a new decoder with the provided maximum dynamic
|
|
// table size. The emitFunc will be called for each valid field
|
|
// parsed, in the same goroutine as calls to Write, before Write returns.
|
|
func NewDecoder(maxDynamicTableSize uint32, emitFunc func(f HeaderField)) *Decoder {
|
|
d := &Decoder{
|
|
emit: emitFunc,
|
|
emitEnabled: true,
|
|
}
|
|
d.dynTab.table.init()
|
|
d.dynTab.allowedMaxSize = maxDynamicTableSize
|
|
d.dynTab.setMaxSize(maxDynamicTableSize)
|
|
return d
|
|
}
|
|
|
|
// ErrStringLength is returned by Decoder.Write when the max string length
|
|
// (as configured by Decoder.SetMaxStringLength) would be violated.
|
|
var ErrStringLength = errors.New("hpack: string too long")
|
|
|
|
// SetMaxStringLength sets the maximum size of a HeaderField name or
|
|
// value string. If a string exceeds this length (even after any
|
|
// decompression), Write will return ErrStringLength.
|
|
// A value of 0 means unlimited and is the default from NewDecoder.
|
|
func (d *Decoder) SetMaxStringLength(n int) {
|
|
d.maxStrLen = n
|
|
}
|
|
|
|
// SetEmitFunc changes the callback used when new header fields
|
|
// are decoded.
|
|
// It must be non-nil. It does not affect EmitEnabled.
|
|
func (d *Decoder) SetEmitFunc(emitFunc func(f HeaderField)) {
|
|
d.emit = emitFunc
|
|
}
|
|
|
|
// SetEmitEnabled controls whether the emitFunc provided to NewDecoder
|
|
// should be called. The default is true.
|
|
//
|
|
// This facility exists to let servers enforce MAX_HEADER_LIST_SIZE
|
|
// while still decoding and keeping in-sync with decoder state, but
|
|
// without doing unnecessary decompression or generating unnecessary
|
|
// garbage for header fields past the limit.
|
|
func (d *Decoder) SetEmitEnabled(v bool) { d.emitEnabled = v }
|
|
|
|
// EmitEnabled reports whether calls to the emitFunc provided to NewDecoder
|
|
// are currently enabled. The default is true.
|
|
func (d *Decoder) EmitEnabled() bool { return d.emitEnabled }
|
|
|
|
// TODO: add method *Decoder.Reset(maxSize, emitFunc) to let callers re-use Decoders and their
|
|
// underlying buffers for garbage reasons.
|
|
|
|
func (d *Decoder) SetMaxDynamicTableSize(v uint32) {
|
|
d.dynTab.setMaxSize(v)
|
|
}
|
|
|
|
// SetAllowedMaxDynamicTableSize sets the upper bound that the encoded
|
|
// stream (via dynamic table size updates) may set the maximum size
|
|
// to.
|
|
func (d *Decoder) SetAllowedMaxDynamicTableSize(v uint32) {
|
|
d.dynTab.allowedMaxSize = v
|
|
}
|
|
|
|
type dynamicTable struct {
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.2.3.2
|
|
table headerFieldTable
|
|
size uint32 // in bytes
|
|
maxSize uint32 // current maxSize
|
|
allowedMaxSize uint32 // maxSize may go up to this, inclusive
|
|
}
|
|
|
|
func (dt *dynamicTable) setMaxSize(v uint32) {
|
|
dt.maxSize = v
|
|
dt.evict()
|
|
}
|
|
|
|
func (dt *dynamicTable) add(f HeaderField) {
|
|
dt.table.addEntry(f)
|
|
dt.size += f.Size()
|
|
dt.evict()
|
|
}
|
|
|
|
// If we're too big, evict old stuff.
|
|
func (dt *dynamicTable) evict() {
|
|
var n int
|
|
for dt.size > dt.maxSize && n < dt.table.len() {
|
|
dt.size -= dt.table.ents[n].Size()
|
|
n++
|
|
}
|
|
dt.table.evictOldest(n)
|
|
}
|
|
|
|
func (d *Decoder) maxTableIndex() int {
|
|
// This should never overflow. RFC 7540 Section 6.5.2 limits the size of
|
|
// the dynamic table to 2^32 bytes, where each entry will occupy more than
|
|
// one byte. Further, the staticTable has a fixed, small length.
|
|
return d.dynTab.table.len() + staticTable.len()
|
|
}
|
|
|
|
func (d *Decoder) at(i uint64) (hf HeaderField, ok bool) {
|
|
// See Section 2.3.3.
|
|
if i == 0 {
|
|
return
|
|
}
|
|
if i <= uint64(staticTable.len()) {
|
|
return staticTable.ents[i-1], true
|
|
}
|
|
if i > uint64(d.maxTableIndex()) {
|
|
return
|
|
}
|
|
// In the dynamic table, newer entries have lower indices.
|
|
// However, dt.ents[0] is the oldest entry. Hence, dt.ents is
|
|
// the reversed dynamic table.
|
|
dt := d.dynTab.table
|
|
return dt.ents[dt.len()-(int(i)-staticTable.len())], true
|
|
}
|
|
|
|
// Decode decodes an entire block.
|
|
//
|
|
// TODO: remove this method and make it incremental later? This is
|
|
// easier for debugging now.
|
|
func (d *Decoder) DecodeFull(p []byte) ([]HeaderField, error) {
|
|
var hf []HeaderField
|
|
saveFunc := d.emit
|
|
defer func() { d.emit = saveFunc }()
|
|
d.emit = func(f HeaderField) { hf = append(hf, f) }
|
|
if _, err := d.Write(p); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := d.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
return hf, nil
|
|
}
|
|
|
|
func (d *Decoder) Close() error {
|
|
if d.saveBuf.Len() > 0 {
|
|
d.saveBuf.Reset()
|
|
return DecodingError{errors.New("truncated headers")}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *Decoder) Write(p []byte) (n int, err error) {
|
|
if len(p) == 0 {
|
|
// Prevent state machine CPU attacks (making us redo
|
|
// work up to the point of finding out we don't have
|
|
// enough data)
|
|
return
|
|
}
|
|
// Only copy the data if we have to. Optimistically assume
|
|
// that p will contain a complete header block.
|
|
if d.saveBuf.Len() == 0 {
|
|
d.buf = p
|
|
} else {
|
|
d.saveBuf.Write(p)
|
|
d.buf = d.saveBuf.Bytes()
|
|
d.saveBuf.Reset()
|
|
}
|
|
|
|
for len(d.buf) > 0 {
|
|
err = d.parseHeaderFieldRepr()
|
|
if err == errNeedMore {
|
|
// Extra paranoia, making sure saveBuf won't
|
|
// get too large. All the varint and string
|
|
// reading code earlier should already catch
|
|
// overlong things and return ErrStringLength,
|
|
// but keep this as a last resort.
|
|
const varIntOverhead = 8 // conservative
|
|
if d.maxStrLen != 0 && int64(len(d.buf)) > 2*(int64(d.maxStrLen)+varIntOverhead) {
|
|
return 0, ErrStringLength
|
|
}
|
|
d.saveBuf.Write(d.buf)
|
|
return len(p), nil
|
|
}
|
|
if err != nil {
|
|
break
|
|
}
|
|
}
|
|
return len(p), err
|
|
}
|
|
|
|
// errNeedMore is an internal sentinel error value that means the
|
|
// buffer is truncated and we need to read more data before we can
|
|
// continue parsing.
|
|
var errNeedMore = errors.New("need more data")
|
|
|
|
type indexType int
|
|
|
|
const (
|
|
indexedTrue indexType = iota
|
|
indexedFalse
|
|
indexedNever
|
|
)
|
|
|
|
func (v indexType) indexed() bool { return v == indexedTrue }
|
|
func (v indexType) sensitive() bool { return v == indexedNever }
|
|
|
|
// returns errNeedMore if there isn't enough data available.
|
|
// any other error is fatal.
|
|
// consumes d.buf iff it returns nil.
|
|
// precondition: must be called with len(d.buf) > 0
|
|
func (d *Decoder) parseHeaderFieldRepr() error {
|
|
b := d.buf[0]
|
|
switch {
|
|
case b&128 != 0:
|
|
// Indexed representation.
|
|
// High bit set?
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.6.1
|
|
return d.parseFieldIndexed()
|
|
case b&192 == 64:
|
|
// 6.2.1 Literal Header Field with Incremental Indexing
|
|
// 0b10xxxxxx: top two bits are 10
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.6.2.1
|
|
return d.parseFieldLiteral(6, indexedTrue)
|
|
case b&240 == 0:
|
|
// 6.2.2 Literal Header Field without Indexing
|
|
// 0b0000xxxx: top four bits are 0000
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.6.2.2
|
|
return d.parseFieldLiteral(4, indexedFalse)
|
|
case b&240 == 16:
|
|
// 6.2.3 Literal Header Field never Indexed
|
|
// 0b0001xxxx: top four bits are 0001
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.6.2.3
|
|
return d.parseFieldLiteral(4, indexedNever)
|
|
case b&224 == 32:
|
|
// 6.3 Dynamic Table Size Update
|
|
// Top three bits are '001'.
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.6.3
|
|
return d.parseDynamicTableSizeUpdate()
|
|
}
|
|
|
|
return DecodingError{errors.New("invalid encoding")}
|
|
}
|
|
|
|
// (same invariants and behavior as parseHeaderFieldRepr)
|
|
func (d *Decoder) parseFieldIndexed() error {
|
|
buf := d.buf
|
|
idx, buf, err := readVarInt(7, buf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
hf, ok := d.at(idx)
|
|
if !ok {
|
|
return DecodingError{InvalidIndexError(idx)}
|
|
}
|
|
d.buf = buf
|
|
return d.callEmit(HeaderField{Name: hf.Name, Value: hf.Value})
|
|
}
|
|
|
|
// (same invariants and behavior as parseHeaderFieldRepr)
|
|
func (d *Decoder) parseFieldLiteral(n uint8, it indexType) error {
|
|
buf := d.buf
|
|
nameIdx, buf, err := readVarInt(n, buf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var hf HeaderField
|
|
wantStr := d.emitEnabled || it.indexed()
|
|
if nameIdx > 0 {
|
|
ihf, ok := d.at(nameIdx)
|
|
if !ok {
|
|
return DecodingError{InvalidIndexError(nameIdx)}
|
|
}
|
|
hf.Name = ihf.Name
|
|
} else {
|
|
hf.Name, buf, err = d.readString(buf, wantStr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
hf.Value, buf, err = d.readString(buf, wantStr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
d.buf = buf
|
|
if it.indexed() {
|
|
d.dynTab.add(hf)
|
|
}
|
|
hf.Sensitive = it.sensitive()
|
|
return d.callEmit(hf)
|
|
}
|
|
|
|
func (d *Decoder) callEmit(hf HeaderField) error {
|
|
if d.maxStrLen != 0 {
|
|
if len(hf.Name) > d.maxStrLen || len(hf.Value) > d.maxStrLen {
|
|
return ErrStringLength
|
|
}
|
|
}
|
|
if d.emitEnabled {
|
|
d.emit(hf)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// (same invariants and behavior as parseHeaderFieldRepr)
|
|
func (d *Decoder) parseDynamicTableSizeUpdate() error {
|
|
buf := d.buf
|
|
size, buf, err := readVarInt(5, buf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if size > uint64(d.dynTab.allowedMaxSize) {
|
|
return DecodingError{errors.New("dynamic table size update too large")}
|
|
}
|
|
d.dynTab.setMaxSize(uint32(size))
|
|
d.buf = buf
|
|
return nil
|
|
}
|
|
|
|
var errVarintOverflow = DecodingError{errors.New("varint integer overflow")}
|
|
|
|
// readVarInt reads an unsigned variable length integer off the
|
|
// beginning of p. n is the parameter as described in
|
|
// http://http2.github.io/http2-spec/compression.html#rfc.section.5.1.
|
|
//
|
|
// n must always be between 1 and 8.
|
|
//
|
|
// The returned remain buffer is either a smaller suffix of p, or err != nil.
|
|
// The error is errNeedMore if p doesn't contain a complete integer.
|
|
func readVarInt(n byte, p []byte) (i uint64, remain []byte, err error) {
|
|
if n < 1 || n > 8 {
|
|
panic("bad n")
|
|
}
|
|
if len(p) == 0 {
|
|
return 0, p, errNeedMore
|
|
}
|
|
i = uint64(p[0])
|
|
if n < 8 {
|
|
i &= (1 << uint64(n)) - 1
|
|
}
|
|
if i < (1<<uint64(n))-1 {
|
|
return i, p[1:], nil
|
|
}
|
|
|
|
origP := p
|
|
p = p[1:]
|
|
var m uint64
|
|
for len(p) > 0 {
|
|
b := p[0]
|
|
p = p[1:]
|
|
i += uint64(b&127) << m
|
|
if b&128 == 0 {
|
|
return i, p, nil
|
|
}
|
|
m += 7
|
|
if m >= 63 { // TODO: proper overflow check. making this up.
|
|
return 0, origP, errVarintOverflow
|
|
}
|
|
}
|
|
return 0, origP, errNeedMore
|
|
}
|
|
|
|
// readString decodes an hpack string from p.
|
|
//
|
|
// wantStr is whether s will be used. If false, decompression and
|
|
// []byte->string garbage are skipped if s will be ignored
|
|
// anyway. This does mean that huffman decoding errors for non-indexed
|
|
// strings past the MAX_HEADER_LIST_SIZE are ignored, but the server
|
|
// is returning an error anyway, and because they're not indexed, the error
|
|
// won't affect the decoding state.
|
|
func (d *Decoder) readString(p []byte, wantStr bool) (s string, remain []byte, err error) {
|
|
if len(p) == 0 {
|
|
return "", p, errNeedMore
|
|
}
|
|
isHuff := p[0]&128 != 0
|
|
strLen, p, err := readVarInt(7, p)
|
|
if err != nil {
|
|
return "", p, err
|
|
}
|
|
if d.maxStrLen != 0 && strLen > uint64(d.maxStrLen) {
|
|
return "", nil, ErrStringLength
|
|
}
|
|
if uint64(len(p)) < strLen {
|
|
return "", p, errNeedMore
|
|
}
|
|
if !isHuff {
|
|
if wantStr {
|
|
s = string(p[:strLen])
|
|
}
|
|
return s, p[strLen:], nil
|
|
}
|
|
|
|
if wantStr {
|
|
buf := bufPool.Get().(*bytes.Buffer)
|
|
buf.Reset() // don't trust others
|
|
defer bufPool.Put(buf)
|
|
if err := huffmanDecode(buf, d.maxStrLen, p[:strLen]); err != nil {
|
|
buf.Reset()
|
|
return "", nil, err
|
|
}
|
|
s = buf.String()
|
|
buf.Reset() // be nice to GC
|
|
}
|
|
return s, p[strLen:], nil
|
|
}
|