diff --git a/cmd/ipfs-cluster-service/pebble_disabled.go b/cmd/ipfs-cluster-service/pebble_disabled.go index 5da3b4af..331a5ee6 100644 --- a/cmd/ipfs-cluster-service/pebble_disabled.go +++ b/cmd/ipfs-cluster-service/pebble_disabled.go @@ -3,6 +3,6 @@ package main const ( - defaultDatastore = "badger" + defaultDatastore = "badger3" datastoreFlagUsage = "select datastore: 'badger', 'badger3' or 'leveldb'" ) diff --git a/cmd/ipfs-cluster-service/pebble_enabled.go b/cmd/ipfs-cluster-service/pebble_enabled.go index ae54ff04..d73a584c 100644 --- a/cmd/ipfs-cluster-service/pebble_enabled.go +++ b/cmd/ipfs-cluster-service/pebble_enabled.go @@ -3,6 +3,6 @@ package main const ( - defaultDatastore = "badger" + defaultDatastore = "pebble" datastoreFlagUsage = "select datastore: 'badger', 'badger3', 'leveldb' or 'pebble'" ) diff --git a/datastore/pebble/config.go b/datastore/pebble/config.go index 70f66460..d1bf3546 100644 --- a/datastore/pebble/config.go +++ b/datastore/pebble/config.go @@ -27,12 +27,52 @@ const ( var ( // DefaultPebbleOptions for convenience. DefaultPebbleOptions pebble.Options + // DefaultCacheSize sets the maximum size of the block cache. + DefaultCacheSize int64 = 1 << 30 // Pebble's default: 8MiB + // DefaultMemTableSize sets the size of the memtables and affecst total + // size of the WAL. It must be under 4GB. + DefaultMemTableSize int = 64 << 20 // Pebble's default: 4MiB + // DefaultMemTableStopWritesThreshold defines how many memtables can + // be queued for writing before stopping writes (memtable memory + // consumption should approach + // MemTableStopWritesThreshold*MemTableSize in that case). + DefaultMemTableStopWritesThreshold = 20 // Pebble's default: 12 + // DefaultBytesPerSync controls how often to call the Filesystem + // Sync. + DefaultBytesPerSync = 1 << 20 // Pebble's default: 512KiB + // DefaultMaxConcurrentCompactions controls how many compactions + // happen at a single time. + DefaultMaxConcurrentCompactions = 5 // Pebble's default: 1 + // DefaultMaxOpenFiles controls how many files can be kept open by + // Pebble. + DefaultMaxOpenFiles = 1000 // Pebble's default: 500 + // DefaultL0CompactionThreshold defines the read amplification on L0 + // that triggers compaction + DefaultL0CompactionThreshold = 4 // Pebble's default: 4 + // DefaultL0CompactionFileThreshold defines the number of files that + // trigger compactions of L0 + DefaultL0CompactionFileThreshold = 750 // Pebble's default: 500 + // DefaultL0StopWritesThreshold defines the critical threshold for + // read amplification on L0, which stops writes until compaction + // reduces it. + DefaultL0StopWritesThreshold = 12 // Pebble's default : 4 + // DefaultLBaseMaxBytes defines maximum size of LBase, where memtables + // are temporally written to + DefaultLBaseMaxBytes int64 = 128 << 20 // Pebble's default: 64MiB + // DefaultL0TargetFileSize defines the target filesize for L0. It is + // multiplied by 2 for every subsequent level. + DefaultL0TargetFileSize int64 = 4 << 20 // Pebble's default: 4M + // DefaultBlockSize defines the target size for table blocks (used in + // all levels). + DefaultBlockSize int = 4 << 10 // Pebble's default: 4KiB + // DefaultFilterPolicy defines the number of bits used per key for + // bloom filters. 10 yields a 1% false positive rate. + DefaultFilterPolicy bloom.FilterPolicy = 10 // Pebble's default: 10 + ) func init() { DefaultPebbleOptions = *DefaultPebbleOptions.EnsureDefaults() - DefaultPebbleOptions.Levels[0].Compression = pebble.NoCompression - DefaultPebbleOptions.Levels[0].FilterPolicy = bloom.FilterPolicy(10) } // Config is used to initialize a Pebble datastore. It implements the @@ -50,6 +90,7 @@ type Config struct { // pebbleOptions is a subset of pebble.Options so it can be marshaled by us in // the cluster configuration. type pebbleOptions struct { + CacheSizeBytes int64 `json:"cache_size_bytes"` BytesPerSync int `json:"bytes_per_sync"` DisableWAL bool `json:"disable_wal"` FlushDelayDeleteRange time.Duration `json:"flush_delay_delete_range"` @@ -60,16 +101,20 @@ type pebbleOptions struct { L0CompactionThreshold int `json:"l0_compaction_threshold"` L0StopWritesThreshold int `json:"l0_stop_writes_threshold"` LBaseMaxBytes int64 `json:"l_base_max_bytes"` - Levels []levelOptions `json:"levels"` MaxOpenFiles int `json:"max_open_files"` MemTableSize int `json:"mem_table_size"` MemTableStopWritesThreshold int `json:"mem_table_stop_writes_threshold"` ReadOnly bool `json:"read_only"` WALBytesPerSync int `json:"wal_bytes_per_sync"` + Levels []levelOptions `json:"levels"` } func (po *pebbleOptions) Unmarshal() *pebble.Options { pebbleOpts := &pebble.Options{} + if size := po.CacheSizeBytes; size > 0 { + cache := pebble.NewCache(size) + pebbleOpts.Cache = cache + } pebbleOpts.BytesPerSync = po.BytesPerSync pebbleOpts.DisableWAL = po.DisableWAL pebbleOpts.FlushDelayDeleteRange = po.FlushDelayDeleteRange @@ -92,6 +137,9 @@ func (po *pebbleOptions) Unmarshal() *pebble.Options { } func (po *pebbleOptions) Marshal(pebbleOpts *pebble.Options) { + if pebbleOpts.Cache != nil { + po.CacheSizeBytes = pebbleOpts.Cache.MaxSize() + } po.BytesPerSync = pebbleOpts.BytesPerSync po.DisableWAL = pebbleOpts.DisableWAL po.FlushDelayDeleteRange = pebbleOpts.FlushDelayDeleteRange @@ -119,6 +167,7 @@ type levelOptions struct { BlockSizeThreshold int `json:"block_size_threshold"` Compression pebble.Compression `json:"compression"` FilterType pebble.FilterType `json:"filter_type"` + FilterPolicy bloom.FilterPolicy `json:"filter_policy"` IndexBlockSize int `json:"index_block_size"` TargetFileSize int64 `json:"target_file_size"` } @@ -130,6 +179,7 @@ func (lo *levelOptions) Unmarshal() *pebble.LevelOptions { levelOpts.BlockSizeThreshold = lo.BlockSizeThreshold levelOpts.Compression = lo.Compression levelOpts.FilterType = lo.FilterType + levelOpts.FilterPolicy = bloom.FilterPolicy(lo.FilterPolicy) levelOpts.IndexBlockSize = lo.IndexBlockSize levelOpts.TargetFileSize = lo.TargetFileSize return levelOpts @@ -141,6 +191,11 @@ func (lo *levelOptions) Marshal(levelOpts *pebble.LevelOptions) { lo.BlockSizeThreshold = levelOpts.BlockSizeThreshold lo.Compression = levelOpts.Compression lo.FilterType = levelOpts.FilterType + + if fp, ok := levelOpts.FilterPolicy.(bloom.FilterPolicy); ok { + lo.FilterPolicy = fp + } + lo.IndexBlockSize = levelOpts.IndexBlockSize lo.TargetFileSize = levelOpts.TargetFileSize } @@ -158,9 +213,35 @@ func (cfg *Config) ConfigKey() string { // Default initializes this Config with sensible values. func (cfg *Config) Default() error { cfg.Folder = DefaultSubFolder - cfg.PebbleOptions = DefaultPebbleOptions cfg.PebbleOptions.Logger = logger + cfg.PebbleOptions = DefaultPebbleOptions + cache := pebble.NewCache(DefaultCacheSize) + cfg.PebbleOptions.Cache = cache + cfg.PebbleOptions.MemTableSize = DefaultMemTableSize + cfg.PebbleOptions.MemTableStopWritesThreshold = DefaultMemTableStopWritesThreshold + cfg.PebbleOptions.BytesPerSync = DefaultBytesPerSync + cfg.PebbleOptions.MaxConcurrentCompactions = func() int { return DefaultMaxConcurrentCompactions } + cfg.PebbleOptions.MaxOpenFiles = DefaultMaxOpenFiles + cfg.PebbleOptions.L0CompactionThreshold = DefaultL0CompactionThreshold + cfg.PebbleOptions.L0CompactionFileThreshold = DefaultL0CompactionFileThreshold + cfg.PebbleOptions.L0StopWritesThreshold = DefaultL0StopWritesThreshold + cfg.PebbleOptions.LBaseMaxBytes = DefaultLBaseMaxBytes + + cfg.PebbleOptions.Levels = make([]pebble.LevelOptions, 7) + cfg.PebbleOptions.Levels[0].TargetFileSize = DefaultL0TargetFileSize + for i := 0; i < len(cfg.PebbleOptions.Levels); i++ { + l := &cfg.PebbleOptions.Levels[i] + l.BlockSize = DefaultBlockSize + l.FilterPolicy = DefaultFilterPolicy + l.FilterType = pebble.TableFilter + if i > 0 { + l.TargetFileSize = cfg.PebbleOptions.Levels[i-1].TargetFileSize * 2 + } + l.EnsureDefaults() // does not overwite, only sets the rest. + + } + return nil } diff --git a/datastore/pebble/pebble.go b/datastore/pebble/pebble.go index a0acbbbf..fd8132ca 100644 --- a/datastore/pebble/pebble.go +++ b/datastore/pebble/pebble.go @@ -5,7 +5,9 @@ package pebble import ( + "context" "os" + "time" ds "github.com/ipfs/go-datastore" pebbleds "github.com/ipfs/go-ds-pebble" @@ -23,7 +25,23 @@ func New(cfg *Config) (ds.Datastore, error) { if err != nil { return nil, errors.Wrap(err, "creating pebble folder") } - return pebbleds.NewDatastore(folder, &cfg.PebbleOptions) + + db, err := pebbleds.NewDatastore(folder, &cfg.PebbleOptions) + if err != nil { + return nil, err + } + + // Calling regularly DB's DiskUsage is a way to printout debug + // database statistics. + go func() { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + <-ticker.C + db.DiskUsage(context.Background()) + } + }() + return db, nil } // Cleanup deletes the badger datastore. diff --git a/logging.go b/logging.go index 6842653c..42efa35d 100644 --- a/logging.go +++ b/logging.go @@ -44,6 +44,7 @@ var LoggingFacilitiesExtra = map[string]string{ "raftlib": "ERROR", "badger": "INFO", "badger3": "INFO", + "pebble": "INFO", } // SetFacilityLogLevel sets the log level for a given module