mutcask

package module

v0.2.9 Latest Latest Go to latest Published: Feb 13, 2023 License: MIT Imports: 23 Imported by: 2

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/filedag-project/mutcask

Links

Open Source Insights

README ¶

Mutcask

Store data chunk in kv style for files and objects.

data storing process

Mutcask has one active write process accepting key-value data. Received data chunk will be append to a log file which has a max size setting by configs. The size of the log files may not as exactly as size in setting, it may be bigger. Once a log file reached the size limit, it sealed, and a new log file will be create to accepting chunks.

As data only be appended to log files, there no rewrite to log files. So we can accept multiple read to one log file.

Documentation ¶

Index ¶

Constants
Variables
func DecodeValue(buf []byte, verify bool) (v []byte, err error)
func EncodeValue(v []byte) []byte
func NewCachedMutcask(cache_num int, opts ...Option) (*cachedMutcask, error)
func NewLevedbKV(dir string) (*levedbKV, error)
func NewMutcask(opts ...Option) (*mutcask, error)
type Cask
- func NewCask(id uint32, kdb *leveldb.DB) *Cask
- func (c *Cask) Close()
- func (c *Cask) Delete(key string) (err error)
- func (c *Cask) Put(key string, value []byte) (err error)
- func (c *Cask) Size(key string) (int, error)
type CaskMap
- func (cm *CaskMap) Add(id uint32, cask *Cask)
- func (cm *CaskMap) CloseAll()
- func (cm *CaskMap) Get(id uint32) (c *Cask, b bool)
type Config
type Hint
- func (h *Hint) Encode() (ret []byte, err error)
- func (h *Hint) From(buf []byte) (err error)
- func (h *Hint) Less(than btree.Item) bool
type HintLV
- func HintLVFromBytes(b []byte) (h *HintLV, err error)
- func (h *HintLV) Bytes() (ret []byte, err error)
type KVDB
- func NewMemkv() KVDB
type KeyMap
- func (km *KeyMap) Add(key string, hint *Hint)
- func (km *KeyMap) Get(key string) (h *Hint, b bool)
type Option
- func CaskNumConf(caskNum int) Option
- func HintBootReadNumConf(hn int) Option
- func MaxParallelReadConf(n int) Option
- func MigrateConf() Option
- func PathConf(dir string) Option

Constants ¶

View Source

const (
	HintDeletedFlag = byte(1)
)

View Source

const HintEncodeSize = HintKeySize + 8 + 4

HintKeySize + 8 bytes value offset + 4 bytes value size

View Source

const HintKeySize = MaxKeySize + 1 + 1

max key size 128 byte + 1 byte which record the key size + 1 byte delete flag

View Source

const MAX_PARALLEL_READ = 56

View Source

const MaxKeySize = 128

Variables ¶

View Source

var (
	ErrNone                = xerrors.New("mutcask: error none")
	ErrValueFormat         = xerrors.New("mutcask: invalid value format")
	ErrDataRotted          = xerrors.New("mutcask: data may be rotted")
	ErrKeySizeTooLong      = xerrors.New("mutcask: key size is too long")
	ErrHintFormat          = xerrors.New("mutcask: invalid hint format")
	ErrPathUndefined       = xerrors.New("mutcask: should define path within config")
	ErrPath                = xerrors.New("mutcask: path should be directory not file")
	ErrHintLogBroken       = xerrors.New("mutcask: hint log broken")
	ErrReadHintBeyondRange = xerrors.New("mutcask: read hint out of file range")
	ErrRepoLocked          = xerrors.New("mutcask: repo has been locked")
)

View Source

var ErrNotFound = xerrors.New("kv: key not found")

Functions ¶

func DecodeValue ¶

func DecodeValue(buf []byte, verify bool) (v []byte, err error)

func EncodeValue ¶

func EncodeValue(v []byte) []byte

*

crc32	:	value
4 		: 	xxxx

*

func NewCachedMutcask ¶ added in v0.1.1

func NewCachedMutcask(cache_num int, opts ...Option) (*cachedMutcask, error)

func NewLevedbKV ¶ added in v0.0.4

func NewLevedbKV(dir string) (*levedbKV, error)

func NewMutcask ¶

func NewMutcask(opts ...Option) (*mutcask, error)

Types ¶

type Cask ¶

type Cask struct {
	// contains filtered or unexported fields
}

func NewCask ¶

func NewCask(id uint32, kdb *leveldb.DB) *Cask

func (*Cask) Close ¶

func (c *Cask) Close()

func (*Cask) Delete ¶

func (c *Cask) Delete(key string) (err error)

func (*Cask) Put ¶

func (c *Cask) Put(key string, value []byte) (err error)

func (*Cask) Size ¶

func (c *Cask) Size(key string) (int, error)

type CaskMap ¶

type CaskMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func (*CaskMap) Add ¶

func (cm *CaskMap) Add(id uint32, cask *Cask)

func (*CaskMap) CloseAll ¶

func (cm *CaskMap) CloseAll()

func (*CaskMap) Get ¶

func (cm *CaskMap) Get(id uint32) (c *Cask, b bool)

type Config ¶

type Config struct {
	Path            string
	CaskNum         uint32
	HintBootReadNum int
	InitBuf         int
	Migrate         bool
	MaxLogFileSize  int
	MaxParallelRead int
}

type Hint ¶

type Hint struct {
	Key     string
	KOffset uint64
	VOffset uint64
	VSize   uint32
	Deleted bool
}

func (*Hint) Encode ¶

func (h *Hint) Encode() (ret []byte, err error)

*

key		:	value offset	:	value size
128+1   :   8   			:   4

*

func (*Hint) From ¶

func (h *Hint) From(buf []byte) (err error)

func (*Hint) Less ¶ added in v0.0.4

func (h *Hint) Less(than btree.Item) bool

type HintLV ¶ added in v0.1.0

type HintLV struct {
	VOffset uint64
	VSize   uint32
}

func HintLVFromBytes ¶ added in v0.1.0

func HintLVFromBytes(b []byte) (h *HintLV, err error)

func (*HintLV) Bytes ¶ added in v0.1.0

func (h *HintLV) Bytes() (ret []byte, err error)

type KVDB ¶

type KVDB interface {
	Put(string, []byte) error
	Delete(string) error
	Get(string) ([]byte, error)
	Size(string) (int, error)
	CheckSum(string) (string, error)

	AllKeysChan(context.Context) (chan string, error)
	Close() error
}

func NewMemkv ¶ added in v0.0.2

func NewMemkv() KVDB

type KeyMap ¶

type KeyMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func (*KeyMap) Add ¶

func (km *KeyMap) Add(key string, hint *Hint)

func (*KeyMap) Get ¶

func (km *KeyMap) Get(key string) (h *Hint, b bool)

type Option ¶

type Option func(cfg *Config)

func CaskNumConf ¶

func CaskNumConf(caskNum int) Option

func HintBootReadNumConf ¶

func HintBootReadNumConf(hn int) Option

func MaxParallelReadConf ¶ added in v0.2.7

func MaxParallelReadConf(n int) Option

func MigrateConf ¶ added in v0.1.0

func MigrateConf() Option

func PathConf ¶

func PathConf(dir string) Option

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL