|
|
|
@ -27,12 +27,52 @@ const (
|
|
|
|
|
var (
|
|
|
|
|
// DefaultPebbleOptions for convenience.
|
|
|
|
|
DefaultPebbleOptions pebble.Options
|
|
|
|
|
// DefaultCacheSize sets the maximum size of the block cache.
|
|
|
|
|
DefaultCacheSize int64 = 1 << 30 // Pebble's default: 8MiB
|
|
|
|
|
// DefaultMemTableSize sets the size of the memtables and affecst total
|
|
|
|
|
// size of the WAL. It must be under 4GB.
|
|
|
|
|
DefaultMemTableSize int = 64 << 20 // Pebble's default: 4MiB
|
|
|
|
|
// DefaultMemTableStopWritesThreshold defines how many memtables can
|
|
|
|
|
// be queued for writing before stopping writes (memtable memory
|
|
|
|
|
// consumption should approach
|
|
|
|
|
// MemTableStopWritesThreshold*MemTableSize in that case).
|
|
|
|
|
DefaultMemTableStopWritesThreshold = 20 // Pebble's default: 12
|
|
|
|
|
// DefaultBytesPerSync controls how often to call the Filesystem
|
|
|
|
|
// Sync.
|
|
|
|
|
DefaultBytesPerSync = 1 << 20 // Pebble's default: 512KiB
|
|
|
|
|
// DefaultMaxConcurrentCompactions controls how many compactions
|
|
|
|
|
// happen at a single time.
|
|
|
|
|
DefaultMaxConcurrentCompactions = 5 // Pebble's default: 1
|
|
|
|
|
// DefaultMaxOpenFiles controls how many files can be kept open by
|
|
|
|
|
// Pebble.
|
|
|
|
|
DefaultMaxOpenFiles = 1000 // Pebble's default: 500
|
|
|
|
|
// DefaultL0CompactionThreshold defines the read amplification on L0
|
|
|
|
|
// that triggers compaction
|
|
|
|
|
DefaultL0CompactionThreshold = 4 // Pebble's default: 4
|
|
|
|
|
// DefaultL0CompactionFileThreshold defines the number of files that
|
|
|
|
|
// trigger compactions of L0
|
|
|
|
|
DefaultL0CompactionFileThreshold = 750 // Pebble's default: 500
|
|
|
|
|
// DefaultL0StopWritesThreshold defines the critical threshold for
|
|
|
|
|
// read amplification on L0, which stops writes until compaction
|
|
|
|
|
// reduces it.
|
|
|
|
|
DefaultL0StopWritesThreshold = 12 // Pebble's default : 4
|
|
|
|
|
// DefaultLBaseMaxBytes defines maximum size of LBase, where memtables
|
|
|
|
|
// are temporally written to
|
|
|
|
|
DefaultLBaseMaxBytes int64 = 128 << 20 // Pebble's default: 64MiB
|
|
|
|
|
// DefaultL0TargetFileSize defines the target filesize for L0. It is
|
|
|
|
|
// multiplied by 2 for every subsequent level.
|
|
|
|
|
DefaultL0TargetFileSize int64 = 4 << 20 // Pebble's default: 4M
|
|
|
|
|
// DefaultBlockSize defines the target size for table blocks (used in
|
|
|
|
|
// all levels).
|
|
|
|
|
DefaultBlockSize int = 4 << 10 // Pebble's default: 4KiB
|
|
|
|
|
// DefaultFilterPolicy defines the number of bits used per key for
|
|
|
|
|
// bloom filters. 10 yields a 1% false positive rate.
|
|
|
|
|
DefaultFilterPolicy bloom.FilterPolicy = 10 // Pebble's default: 10
|
|
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
|
DefaultPebbleOptions = *DefaultPebbleOptions.EnsureDefaults()
|
|
|
|
|
DefaultPebbleOptions.Levels[0].Compression = pebble.NoCompression
|
|
|
|
|
DefaultPebbleOptions.Levels[0].FilterPolicy = bloom.FilterPolicy(10)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Config is used to initialize a Pebble datastore. It implements the
|
|
|
|
@ -50,6 +90,7 @@ type Config struct {
|
|
|
|
|
// pebbleOptions is a subset of pebble.Options so it can be marshaled by us in
|
|
|
|
|
// the cluster configuration.
|
|
|
|
|
type pebbleOptions struct {
|
|
|
|
|
CacheSizeBytes int64 `json:"cache_size_bytes"`
|
|
|
|
|
BytesPerSync int `json:"bytes_per_sync"`
|
|
|
|
|
DisableWAL bool `json:"disable_wal"`
|
|
|
|
|
FlushDelayDeleteRange time.Duration `json:"flush_delay_delete_range"`
|
|
|
|
@ -60,16 +101,20 @@ type pebbleOptions struct {
|
|
|
|
|
L0CompactionThreshold int `json:"l0_compaction_threshold"`
|
|
|
|
|
L0StopWritesThreshold int `json:"l0_stop_writes_threshold"`
|
|
|
|
|
LBaseMaxBytes int64 `json:"l_base_max_bytes"`
|
|
|
|
|
Levels []levelOptions `json:"levels"`
|
|
|
|
|
MaxOpenFiles int `json:"max_open_files"`
|
|
|
|
|
MemTableSize int `json:"mem_table_size"`
|
|
|
|
|
MemTableStopWritesThreshold int `json:"mem_table_stop_writes_threshold"`
|
|
|
|
|
ReadOnly bool `json:"read_only"`
|
|
|
|
|
WALBytesPerSync int `json:"wal_bytes_per_sync"`
|
|
|
|
|
Levels []levelOptions `json:"levels"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (po *pebbleOptions) Unmarshal() *pebble.Options {
|
|
|
|
|
pebbleOpts := &pebble.Options{}
|
|
|
|
|
if size := po.CacheSizeBytes; size > 0 {
|
|
|
|
|
cache := pebble.NewCache(size)
|
|
|
|
|
pebbleOpts.Cache = cache
|
|
|
|
|
}
|
|
|
|
|
pebbleOpts.BytesPerSync = po.BytesPerSync
|
|
|
|
|
pebbleOpts.DisableWAL = po.DisableWAL
|
|
|
|
|
pebbleOpts.FlushDelayDeleteRange = po.FlushDelayDeleteRange
|
|
|
|
@ -92,6 +137,9 @@ func (po *pebbleOptions) Unmarshal() *pebble.Options {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func (po *pebbleOptions) Marshal(pebbleOpts *pebble.Options) {
|
|
|
|
|
if pebbleOpts.Cache != nil {
|
|
|
|
|
po.CacheSizeBytes = pebbleOpts.Cache.MaxSize()
|
|
|
|
|
}
|
|
|
|
|
po.BytesPerSync = pebbleOpts.BytesPerSync
|
|
|
|
|
po.DisableWAL = pebbleOpts.DisableWAL
|
|
|
|
|
po.FlushDelayDeleteRange = pebbleOpts.FlushDelayDeleteRange
|
|
|
|
@ -119,6 +167,7 @@ type levelOptions struct {
|
|
|
|
|
BlockSizeThreshold int `json:"block_size_threshold"`
|
|
|
|
|
Compression pebble.Compression `json:"compression"`
|
|
|
|
|
FilterType pebble.FilterType `json:"filter_type"`
|
|
|
|
|
FilterPolicy bloom.FilterPolicy `json:"filter_policy"`
|
|
|
|
|
IndexBlockSize int `json:"index_block_size"`
|
|
|
|
|
TargetFileSize int64 `json:"target_file_size"`
|
|
|
|
|
}
|
|
|
|
@ -130,6 +179,7 @@ func (lo *levelOptions) Unmarshal() *pebble.LevelOptions {
|
|
|
|
|
levelOpts.BlockSizeThreshold = lo.BlockSizeThreshold
|
|
|
|
|
levelOpts.Compression = lo.Compression
|
|
|
|
|
levelOpts.FilterType = lo.FilterType
|
|
|
|
|
levelOpts.FilterPolicy = bloom.FilterPolicy(lo.FilterPolicy)
|
|
|
|
|
levelOpts.IndexBlockSize = lo.IndexBlockSize
|
|
|
|
|
levelOpts.TargetFileSize = lo.TargetFileSize
|
|
|
|
|
return levelOpts
|
|
|
|
@ -141,6 +191,11 @@ func (lo *levelOptions) Marshal(levelOpts *pebble.LevelOptions) {
|
|
|
|
|
lo.BlockSizeThreshold = levelOpts.BlockSizeThreshold
|
|
|
|
|
lo.Compression = levelOpts.Compression
|
|
|
|
|
lo.FilterType = levelOpts.FilterType
|
|
|
|
|
|
|
|
|
|
if fp, ok := levelOpts.FilterPolicy.(bloom.FilterPolicy); ok {
|
|
|
|
|
lo.FilterPolicy = fp
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
lo.IndexBlockSize = levelOpts.IndexBlockSize
|
|
|
|
|
lo.TargetFileSize = levelOpts.TargetFileSize
|
|
|
|
|
}
|
|
|
|
@ -158,9 +213,35 @@ func (cfg *Config) ConfigKey() string {
|
|
|
|
|
// Default initializes this Config with sensible values.
|
|
|
|
|
func (cfg *Config) Default() error {
|
|
|
|
|
cfg.Folder = DefaultSubFolder
|
|
|
|
|
cfg.PebbleOptions = DefaultPebbleOptions
|
|
|
|
|
cfg.PebbleOptions.Logger = logger
|
|
|
|
|
|
|
|
|
|
cfg.PebbleOptions = DefaultPebbleOptions
|
|
|
|
|
cache := pebble.NewCache(DefaultCacheSize)
|
|
|
|
|
cfg.PebbleOptions.Cache = cache
|
|
|
|
|
cfg.PebbleOptions.MemTableSize = DefaultMemTableSize
|
|
|
|
|
cfg.PebbleOptions.MemTableStopWritesThreshold = DefaultMemTableStopWritesThreshold
|
|
|
|
|
cfg.PebbleOptions.BytesPerSync = DefaultBytesPerSync
|
|
|
|
|
cfg.PebbleOptions.MaxConcurrentCompactions = func() int { return DefaultMaxConcurrentCompactions }
|
|
|
|
|
cfg.PebbleOptions.MaxOpenFiles = DefaultMaxOpenFiles
|
|
|
|
|
cfg.PebbleOptions.L0CompactionThreshold = DefaultL0CompactionThreshold
|
|
|
|
|
cfg.PebbleOptions.L0CompactionFileThreshold = DefaultL0CompactionFileThreshold
|
|
|
|
|
cfg.PebbleOptions.L0StopWritesThreshold = DefaultL0StopWritesThreshold
|
|
|
|
|
cfg.PebbleOptions.LBaseMaxBytes = DefaultLBaseMaxBytes
|
|
|
|
|
|
|
|
|
|
cfg.PebbleOptions.Levels = make([]pebble.LevelOptions, 7)
|
|
|
|
|
cfg.PebbleOptions.Levels[0].TargetFileSize = DefaultL0TargetFileSize
|
|
|
|
|
for i := 0; i < len(cfg.PebbleOptions.Levels); i++ {
|
|
|
|
|
l := &cfg.PebbleOptions.Levels[i]
|
|
|
|
|
l.BlockSize = DefaultBlockSize
|
|
|
|
|
l.FilterPolicy = DefaultFilterPolicy
|
|
|
|
|
l.FilterType = pebble.TableFilter
|
|
|
|
|
if i > 0 {
|
|
|
|
|
l.TargetFileSize = cfg.PebbleOptions.Levels[i-1].TargetFileSize * 2
|
|
|
|
|
}
|
|
|
|
|
l.EnsureDefaults() // does not overwite, only sets the rest.
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return nil
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|