lib/mmap: library to do memory allocation with anonymous memory maps

This commit is contained in:
Nick Craig-Wood 2018-05-22 14:47:30 +01:00
parent a43ed567ee
commit f0696dfe30
5 changed files with 222 additions and 0 deletions

29
lib/mmap/mmap.go Normal file
View File

@ -0,0 +1,29 @@
package mmap
import "os"
// PageSize is the minimum allocation size. Allocations will use at
// least this size and are likely to be multiplied up to a multiple of
// this size.
var PageSize = os.Getpagesize()
// MustAlloc allocates size bytes and returns a slice containing them. If
// the allocation fails it will panic. This is best used for
// allocations which are a multiple of the PageSize.
func MustAlloc(size int) []byte {
mem, err := Alloc(size)
if err != nil {
panic(err)
}
return mem
}
// MustFree frees buffers allocated by Alloc. Note it should be passed
// the same slice (not a derived slice) that Alloc returned. If the
// free fails it will panic.
func MustFree(mem []byte) {
err := Free(mem)
if err != nil {
panic(err)
}
}

99
lib/mmap/mmap_test.go Normal file
View File

@ -0,0 +1,99 @@
package mmap
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
// Constants to control the benchmarking
const (
maxAllocs = 16 * 1024
)
func TestAllocFree(t *testing.T) {
const Size = 4096
b := MustAlloc(Size)
assert.Equal(t, Size, len(b))
// check we can write to all the memory
for i := range b {
b[i] = byte(i)
}
// Now free the memory
MustFree(b)
}
func BenchmarkAllocFree(b *testing.B) {
for _, dirty := range []bool{false, true} {
for size := 4096; size <= 32*1024*1024; size *= 2 {
b.Run(fmt.Sprintf("%dk,dirty=%v", size>>10, dirty), func(b *testing.B) {
for i := 0; i < b.N; i++ {
mem := MustAlloc(size)
if dirty {
mem[0] ^= 0xFF
}
MustFree(mem)
}
})
}
}
}
// benchmark the time alloc/free takes with lots of allocations already
func BenchmarkAllocFreeWithLotsOfAllocations(b *testing.B) {
const size = 4096
alloc := func(n int) (allocs [][]byte) {
for i := 0; i < n; i++ {
mem := MustAlloc(size)
mem[0] ^= 0xFF
allocs = append(allocs, mem)
}
return allocs
}
free := func(allocs [][]byte) {
for _, mem := range allocs {
MustFree(mem)
}
}
for preAllocs := 1; preAllocs <= maxAllocs; preAllocs *= 2 {
allocs := alloc(preAllocs)
b.Run(fmt.Sprintf("%d", preAllocs), func(b *testing.B) {
for i := 0; i < b.N; i++ {
mem := MustAlloc(size)
mem[0] ^= 0xFF
MustFree(mem)
}
})
free(allocs)
}
}
// benchmark the time alloc/free takes for lots of allocations
func BenchmarkAllocFreeForLotsOfAllocations(b *testing.B) {
const size = 4096
alloc := func(n int) (allocs [][]byte) {
for i := 0; i < n; i++ {
mem := MustAlloc(size)
mem[0] ^= 0xFF
allocs = append(allocs, mem)
}
return allocs
}
free := func(allocs [][]byte) {
for _, mem := range allocs {
MustFree(mem)
}
}
for preAllocs := 1; preAllocs <= maxAllocs; preAllocs *= 2 {
b.Run(fmt.Sprintf("%d", preAllocs), func(b *testing.B) {
for i := 0; i < b.N; i++ {
allocs := alloc(preAllocs)
free(allocs)
}
})
}
}

33
lib/mmap/mmap_unix.go Normal file
View File

@ -0,0 +1,33 @@
// Package mmap implements a large block memory allocator using
// anonymous memory maps.
// +build !plan9,!windows
package mmap
import (
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
// Alloc allocates size bytes and returns a slice containing them. If
// the allocation fails it will return with an error. This is best
// used for allocations which are a multiple of the PageSize.
func Alloc(size int) ([]byte, error) {
mem, err := unix.Mmap(-1, 0, size, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_PRIVATE|unix.MAP_ANON)
if err != nil {
return nil, errors.Wrap(err, "mmap: failed to allocate memory for buffer")
}
return mem, nil
}
// Free frees buffers allocated by Alloc. Note it should be passed
// the same slice (not a derived slice) that Alloc returned. If the
// free fails it will return with an error.
func Free(mem []byte) error {
err := unix.Munmap(mem)
if err != nil {
return errors.Wrap(err, "mmap: failed to unmap memory")
}
return nil
}

View File

@ -0,0 +1,19 @@
// Fallback Alloc and Free for unsupported OSes
// +build plan9
package mmap
// Alloc allocates size bytes and returns a slice containing them. If
// the allocation fails it will return with an error. This is best
// used for allocations which are a multiple of the Pagesize.
func Alloc(size int) ([]byte, error) {
return make([]byte, size), nil
}
// Free frees buffers allocated by Alloc. Note it should be passed
// the same slice (not a derived slice) that Alloc returned. If the
// free fails it will return with an error.
func Free(mem []byte) error {
return nil
}

42
lib/mmap/mmap_windows.go Normal file
View File

@ -0,0 +1,42 @@
// Package mmap implements a large block memory allocator using
// anonymous memory maps.
// +build windows
package mmap
import (
"reflect"
"unsafe"
"github.com/pkg/errors"
"golang.org/x/sys/windows"
)
// Alloc allocates size bytes and returns a slice containing them. If
// the allocation fails it will return with an error. This is best
// used for allocations which are a multiple of the PageSize.
func Alloc(size int) ([]byte, error) {
p, err := windows.VirtualAlloc(0, uintptr(size), windows.MEM_COMMIT, windows.PAGE_READWRITE)
if err != nil {
return nil, errors.Wrap(err, "mmap: failed to allocate memory for buffer")
}
var mem []byte
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem))
sh.Data = p
sh.Len = size
sh.Cap = size
return mem, nil
}
// Free frees buffers allocated by Alloc. Note it should be passed
// the same slice (not a derived slice) that Alloc returned. If the
// free fails it will return with an error.
func Free(mem []byte) error {
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem))
err := windows.VirtualFree(sh.Data, 0, windows.MEM_RELEASE)
if err != nil {
return errors.Wrap(err, "mmap: failed to unmap memory")
}
return nil
}