mirror of
https://github.com/rclone/rclone.git
synced 2024-11-07 09:04:52 +01:00
lib/mmap: library to do memory allocation with anonymous memory maps
This commit is contained in:
parent
a43ed567ee
commit
f0696dfe30
29
lib/mmap/mmap.go
Normal file
29
lib/mmap/mmap.go
Normal file
@ -0,0 +1,29 @@
|
||||
package mmap
|
||||
|
||||
import "os"
|
||||
|
||||
// PageSize is the minimum allocation size. Allocations will use at
|
||||
// least this size and are likely to be multiplied up to a multiple of
|
||||
// this size.
|
||||
var PageSize = os.Getpagesize()
|
||||
|
||||
// MustAlloc allocates size bytes and returns a slice containing them. If
|
||||
// the allocation fails it will panic. This is best used for
|
||||
// allocations which are a multiple of the PageSize.
|
||||
func MustAlloc(size int) []byte {
|
||||
mem, err := Alloc(size)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return mem
|
||||
}
|
||||
|
||||
// MustFree frees buffers allocated by Alloc. Note it should be passed
|
||||
// the same slice (not a derived slice) that Alloc returned. If the
|
||||
// free fails it will panic.
|
||||
func MustFree(mem []byte) {
|
||||
err := Free(mem)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
99
lib/mmap/mmap_test.go
Normal file
99
lib/mmap/mmap_test.go
Normal file
@ -0,0 +1,99 @@
|
||||
package mmap
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Constants to control the benchmarking
|
||||
const (
|
||||
maxAllocs = 16 * 1024
|
||||
)
|
||||
|
||||
func TestAllocFree(t *testing.T) {
|
||||
const Size = 4096
|
||||
|
||||
b := MustAlloc(Size)
|
||||
assert.Equal(t, Size, len(b))
|
||||
|
||||
// check we can write to all the memory
|
||||
for i := range b {
|
||||
b[i] = byte(i)
|
||||
}
|
||||
|
||||
// Now free the memory
|
||||
MustFree(b)
|
||||
}
|
||||
|
||||
func BenchmarkAllocFree(b *testing.B) {
|
||||
for _, dirty := range []bool{false, true} {
|
||||
for size := 4096; size <= 32*1024*1024; size *= 2 {
|
||||
b.Run(fmt.Sprintf("%dk,dirty=%v", size>>10, dirty), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
mem := MustAlloc(size)
|
||||
if dirty {
|
||||
mem[0] ^= 0xFF
|
||||
}
|
||||
MustFree(mem)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// benchmark the time alloc/free takes with lots of allocations already
|
||||
func BenchmarkAllocFreeWithLotsOfAllocations(b *testing.B) {
|
||||
const size = 4096
|
||||
alloc := func(n int) (allocs [][]byte) {
|
||||
for i := 0; i < n; i++ {
|
||||
mem := MustAlloc(size)
|
||||
mem[0] ^= 0xFF
|
||||
allocs = append(allocs, mem)
|
||||
}
|
||||
return allocs
|
||||
}
|
||||
free := func(allocs [][]byte) {
|
||||
for _, mem := range allocs {
|
||||
MustFree(mem)
|
||||
}
|
||||
}
|
||||
for preAllocs := 1; preAllocs <= maxAllocs; preAllocs *= 2 {
|
||||
allocs := alloc(preAllocs)
|
||||
b.Run(fmt.Sprintf("%d", preAllocs), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
mem := MustAlloc(size)
|
||||
mem[0] ^= 0xFF
|
||||
MustFree(mem)
|
||||
}
|
||||
})
|
||||
free(allocs)
|
||||
}
|
||||
}
|
||||
|
||||
// benchmark the time alloc/free takes for lots of allocations
|
||||
func BenchmarkAllocFreeForLotsOfAllocations(b *testing.B) {
|
||||
const size = 4096
|
||||
alloc := func(n int) (allocs [][]byte) {
|
||||
for i := 0; i < n; i++ {
|
||||
mem := MustAlloc(size)
|
||||
mem[0] ^= 0xFF
|
||||
allocs = append(allocs, mem)
|
||||
}
|
||||
return allocs
|
||||
}
|
||||
free := func(allocs [][]byte) {
|
||||
for _, mem := range allocs {
|
||||
MustFree(mem)
|
||||
}
|
||||
}
|
||||
for preAllocs := 1; preAllocs <= maxAllocs; preAllocs *= 2 {
|
||||
b.Run(fmt.Sprintf("%d", preAllocs), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
allocs := alloc(preAllocs)
|
||||
free(allocs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
33
lib/mmap/mmap_unix.go
Normal file
33
lib/mmap/mmap_unix.go
Normal file
@ -0,0 +1,33 @@
|
||||
// Package mmap implements a large block memory allocator using
|
||||
// anonymous memory maps.
|
||||
|
||||
// +build !plan9,!windows
|
||||
|
||||
package mmap
|
||||
|
||||
import (
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Alloc allocates size bytes and returns a slice containing them. If
|
||||
// the allocation fails it will return with an error. This is best
|
||||
// used for allocations which are a multiple of the PageSize.
|
||||
func Alloc(size int) ([]byte, error) {
|
||||
mem, err := unix.Mmap(-1, 0, size, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_PRIVATE|unix.MAP_ANON)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "mmap: failed to allocate memory for buffer")
|
||||
}
|
||||
return mem, nil
|
||||
}
|
||||
|
||||
// Free frees buffers allocated by Alloc. Note it should be passed
|
||||
// the same slice (not a derived slice) that Alloc returned. If the
|
||||
// free fails it will return with an error.
|
||||
func Free(mem []byte) error {
|
||||
err := unix.Munmap(mem)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "mmap: failed to unmap memory")
|
||||
}
|
||||
return nil
|
||||
}
|
19
lib/mmap/mmap_unsupported.go
Normal file
19
lib/mmap/mmap_unsupported.go
Normal file
@ -0,0 +1,19 @@
|
||||
// Fallback Alloc and Free for unsupported OSes
|
||||
|
||||
// +build plan9
|
||||
|
||||
package mmap
|
||||
|
||||
// Alloc allocates size bytes and returns a slice containing them. If
|
||||
// the allocation fails it will return with an error. This is best
|
||||
// used for allocations which are a multiple of the Pagesize.
|
||||
func Alloc(size int) ([]byte, error) {
|
||||
return make([]byte, size), nil
|
||||
}
|
||||
|
||||
// Free frees buffers allocated by Alloc. Note it should be passed
|
||||
// the same slice (not a derived slice) that Alloc returned. If the
|
||||
// free fails it will return with an error.
|
||||
func Free(mem []byte) error {
|
||||
return nil
|
||||
}
|
42
lib/mmap/mmap_windows.go
Normal file
42
lib/mmap/mmap_windows.go
Normal file
@ -0,0 +1,42 @@
|
||||
// Package mmap implements a large block memory allocator using
|
||||
// anonymous memory maps.
|
||||
|
||||
// +build windows
|
||||
|
||||
package mmap
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"unsafe"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// Alloc allocates size bytes and returns a slice containing them. If
|
||||
// the allocation fails it will return with an error. This is best
|
||||
// used for allocations which are a multiple of the PageSize.
|
||||
func Alloc(size int) ([]byte, error) {
|
||||
p, err := windows.VirtualAlloc(0, uintptr(size), windows.MEM_COMMIT, windows.PAGE_READWRITE)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "mmap: failed to allocate memory for buffer")
|
||||
}
|
||||
var mem []byte
|
||||
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem))
|
||||
sh.Data = p
|
||||
sh.Len = size
|
||||
sh.Cap = size
|
||||
return mem, nil
|
||||
}
|
||||
|
||||
// Free frees buffers allocated by Alloc. Note it should be passed
|
||||
// the same slice (not a derived slice) that Alloc returned. If the
|
||||
// free fails it will return with an error.
|
||||
func Free(mem []byte) error {
|
||||
sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem))
|
||||
err := windows.VirtualFree(sh.Data, 0, windows.MEM_RELEASE)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "mmap: failed to unmap memory")
|
||||
}
|
||||
return nil
|
||||
}
|
Loading…
Reference in New Issue
Block a user