2023-03-03 15:34:53 +00:00
|
|
|
//go:build !arm && !386 && !(openbsd && amd64)
|
|
|
|
|
2022-12-01 15:58:32 +00:00
|
|
|
package pebble
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"path/filepath"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/cockroachdb/pebble"
|
|
|
|
"github.com/cockroachdb/pebble/bloom"
|
|
|
|
"github.com/imdario/mergo"
|
|
|
|
"github.com/kelseyhightower/envconfig"
|
|
|
|
|
|
|
|
"github.com/ipfs-cluster/ipfs-cluster/config"
|
|
|
|
)
|
|
|
|
|
|
|
|
const configKey = "pebble"
|
|
|
|
const envConfigKey = "cluster_pebble"
|
|
|
|
|
|
|
|
// Default values for Pebble Config
|
|
|
|
const (
|
|
|
|
DefaultSubFolder = "pebble"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
// DefaultPebbleOptions for convenience.
|
|
|
|
DefaultPebbleOptions pebble.Options
|
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
DefaultPebbleOptions = *DefaultPebbleOptions.EnsureDefaults()
|
|
|
|
DefaultPebbleOptions.Levels[0].Compression = pebble.NoCompression
|
|
|
|
DefaultPebbleOptions.Levels[0].FilterPolicy = bloom.FilterPolicy(10)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Config is used to initialize a Pebble datastore. It implements the
|
|
|
|
// ComponentConfig interface.
|
|
|
|
type Config struct {
|
|
|
|
config.Saver
|
|
|
|
|
|
|
|
// The folder for this datastore. Non-absolute paths are relative to
|
|
|
|
// the base configuration folder.
|
|
|
|
Folder string
|
|
|
|
|
|
|
|
PebbleOptions pebble.Options
|
|
|
|
}
|
|
|
|
|
|
|
|
// pebbleOptions is a subset of pebble.Options so it can be marshaled by us in
|
|
|
|
// the cluster configuration.
|
|
|
|
type pebbleOptions struct {
|
|
|
|
BytesPerSync int `json:"bytes_per_sync"`
|
|
|
|
DisableWAL bool `json:"disable_wal"`
|
|
|
|
FlushDelayDeleteRange time.Duration `json:"flush_delay_delete_range"`
|
|
|
|
FlushDelayRangeKey time.Duration `json:"flush_delay_range_key"`
|
|
|
|
FlushSplitBytes int64 `json:"flush_split_bytes"`
|
|
|
|
FormatMajorVersion pebble.FormatMajorVersion `json:"format_major_version"`
|
|
|
|
L0CompactionFileThreshold int `json:"l0_compaction_file_threshold"`
|
|
|
|
L0CompactionThreshold int `json:"l0_compaction_threshold"`
|
|
|
|
L0StopWritesThreshold int `json:"l0_stop_writes_threshold"`
|
|
|
|
LBaseMaxBytes int64 `json:"l_base_max_bytes"`
|
|
|
|
Levels []levelOptions `json:"levels"`
|
|
|
|
MaxOpenFiles int `json:"max_open_files"`
|
|
|
|
MemTableSize int `json:"mem_table_size"`
|
|
|
|
MemTableStopWritesThreshold int `json:"mem_table_stop_writes_threshold"`
|
|
|
|
ReadOnly bool `json:"read_only"`
|
|
|
|
WALBytesPerSync int `json:"wal_bytes_per_sync"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (po *pebbleOptions) Unmarshal() *pebble.Options {
|
|
|
|
pebbleOpts := &pebble.Options{}
|
|
|
|
pebbleOpts.BytesPerSync = po.BytesPerSync
|
|
|
|
pebbleOpts.DisableWAL = po.DisableWAL
|
|
|
|
pebbleOpts.FlushDelayDeleteRange = po.FlushDelayDeleteRange
|
|
|
|
pebbleOpts.FlushSplitBytes = po.FlushSplitBytes
|
|
|
|
pebbleOpts.FormatMajorVersion = po.FormatMajorVersion
|
|
|
|
pebbleOpts.L0CompactionFileThreshold = po.L0CompactionFileThreshold
|
|
|
|
pebbleOpts.L0CompactionThreshold = po.L0CompactionThreshold
|
|
|
|
pebbleOpts.L0StopWritesThreshold = po.L0StopWritesThreshold
|
|
|
|
pebbleOpts.LBaseMaxBytes = po.LBaseMaxBytes
|
|
|
|
pebbleOpts.Levels = make([]pebble.LevelOptions, len(po.Levels))
|
|
|
|
for i := range po.Levels {
|
|
|
|
pebbleOpts.Levels[i] = *po.Levels[i].Unmarshal()
|
|
|
|
}
|
|
|
|
pebbleOpts.MaxOpenFiles = po.MaxOpenFiles
|
|
|
|
pebbleOpts.MemTableSize = po.MemTableSize
|
|
|
|
pebbleOpts.MemTableStopWritesThreshold = po.MemTableStopWritesThreshold
|
|
|
|
pebbleOpts.ReadOnly = po.ReadOnly
|
|
|
|
pebbleOpts.WALBytesPerSync = po.WALBytesPerSync
|
|
|
|
return pebbleOpts
|
|
|
|
}
|
|
|
|
|
|
|
|
func (po *pebbleOptions) Marshal(pebbleOpts *pebble.Options) {
|
|
|
|
po.BytesPerSync = pebbleOpts.BytesPerSync
|
|
|
|
po.DisableWAL = pebbleOpts.DisableWAL
|
|
|
|
po.FlushDelayDeleteRange = pebbleOpts.FlushDelayDeleteRange
|
|
|
|
po.FlushSplitBytes = pebbleOpts.FlushSplitBytes
|
|
|
|
po.FormatMajorVersion = pebbleOpts.FormatMajorVersion
|
|
|
|
po.L0CompactionFileThreshold = pebbleOpts.L0CompactionFileThreshold
|
|
|
|
po.L0CompactionThreshold = pebbleOpts.L0CompactionThreshold
|
|
|
|
po.L0StopWritesThreshold = pebbleOpts.L0StopWritesThreshold
|
|
|
|
po.LBaseMaxBytes = pebbleOpts.LBaseMaxBytes
|
|
|
|
po.Levels = make([]levelOptions, len(pebbleOpts.Levels))
|
|
|
|
for i := range pebbleOpts.Levels {
|
|
|
|
po.Levels[i].Marshal(&pebbleOpts.Levels[i])
|
|
|
|
}
|
|
|
|
po.MaxOpenFiles = pebbleOpts.MaxOpenFiles
|
|
|
|
po.MemTableSize = pebbleOpts.MemTableSize
|
|
|
|
po.MemTableStopWritesThreshold = pebbleOpts.MemTableStopWritesThreshold
|
|
|
|
po.ReadOnly = pebbleOpts.ReadOnly
|
|
|
|
po.WALBytesPerSync = pebbleOpts.WALBytesPerSync
|
|
|
|
}
|
|
|
|
|
|
|
|
// levelOptions carries options for pebble's per-level parameters.
|
|
|
|
type levelOptions struct {
|
|
|
|
BlockRestartInterval int `json:"block_restart_interval"`
|
|
|
|
BlockSize int `json:"block_size"`
|
|
|
|
BlockSizeThreshold int `json:"block_size_threshold"`
|
|
|
|
Compression pebble.Compression `json:"compression"`
|
|
|
|
FilterType pebble.FilterType `json:"filter_type"`
|
|
|
|
IndexBlockSize int `json:"index_block_size"`
|
|
|
|
TargetFileSize int64 `json:"target_file_size"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (lo *levelOptions) Unmarshal() *pebble.LevelOptions {
|
|
|
|
levelOpts := &pebble.LevelOptions{}
|
|
|
|
levelOpts.BlockRestartInterval = lo.BlockRestartInterval
|
|
|
|
levelOpts.BlockSize = lo.BlockSize
|
|
|
|
levelOpts.BlockSizeThreshold = lo.BlockSizeThreshold
|
|
|
|
levelOpts.Compression = lo.Compression
|
|
|
|
levelOpts.FilterType = lo.FilterType
|
|
|
|
levelOpts.IndexBlockSize = lo.IndexBlockSize
|
|
|
|
levelOpts.TargetFileSize = lo.TargetFileSize
|
|
|
|
return levelOpts
|
|
|
|
}
|
|
|
|
|
|
|
|
func (lo *levelOptions) Marshal(levelOpts *pebble.LevelOptions) {
|
|
|
|
lo.BlockRestartInterval = levelOpts.BlockRestartInterval
|
|
|
|
lo.BlockSize = levelOpts.BlockSize
|
|
|
|
lo.BlockSizeThreshold = levelOpts.BlockSizeThreshold
|
|
|
|
lo.Compression = levelOpts.Compression
|
|
|
|
lo.FilterType = levelOpts.FilterType
|
|
|
|
lo.IndexBlockSize = levelOpts.IndexBlockSize
|
|
|
|
lo.TargetFileSize = levelOpts.TargetFileSize
|
|
|
|
}
|
|
|
|
|
|
|
|
type jsonConfig struct {
|
|
|
|
Folder string `json:"folder,omitempty"`
|
|
|
|
PebbleOptions pebbleOptions `json:"pebble_options,omitempty"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// ConfigKey returns a human-friendly identifier for this type of Datastore.
|
|
|
|
func (cfg *Config) ConfigKey() string {
|
|
|
|
return configKey
|
|
|
|
}
|
|
|
|
|
|
|
|
// Default initializes this Config with sensible values.
|
|
|
|
func (cfg *Config) Default() error {
|
|
|
|
cfg.Folder = DefaultSubFolder
|
|
|
|
cfg.PebbleOptions = DefaultPebbleOptions
|
|
|
|
cfg.PebbleOptions.Logger = logger
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ApplyEnvVars fills in any Config fields found as environment variables.
|
|
|
|
func (cfg *Config) ApplyEnvVars() error {
|
|
|
|
jcfg := cfg.toJSONConfig()
|
|
|
|
|
|
|
|
err := envconfig.Process(envConfigKey, jcfg)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return cfg.applyJSONConfig(jcfg)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Validate checks that the fields of this Config have working values,
|
|
|
|
// at least in appearance.
|
|
|
|
func (cfg *Config) Validate() error {
|
|
|
|
if cfg.Folder == "" {
|
|
|
|
return errors.New("folder is unset")
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := cfg.PebbleOptions.Validate(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// LoadJSON reads the fields of this Config from a JSON byteslice as
|
|
|
|
// generated by ToJSON.
|
|
|
|
func (cfg *Config) LoadJSON(raw []byte) error {
|
|
|
|
jcfg := &jsonConfig{}
|
|
|
|
err := json.Unmarshal(raw, jcfg)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
cfg.Default()
|
|
|
|
|
|
|
|
return cfg.applyJSONConfig(jcfg)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
|
|
|
|
config.SetIfNotDefault(jcfg.Folder, &cfg.Folder)
|
|
|
|
|
|
|
|
pebbleOpts := jcfg.PebbleOptions.Unmarshal()
|
|
|
|
|
|
|
|
if err := mergo.Merge(&cfg.PebbleOptions, pebbleOpts, mergo.WithOverride); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return cfg.Validate()
|
|
|
|
}
|
|
|
|
|
|
|
|
// ToJSON generates a JSON-formatted human-friendly representation of this
|
|
|
|
// Config.
|
|
|
|
func (cfg *Config) ToJSON() (raw []byte, err error) {
|
|
|
|
jcfg := cfg.toJSONConfig()
|
|
|
|
|
|
|
|
raw, err = config.DefaultJSONMarshal(jcfg)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
func (cfg *Config) toJSONConfig() *jsonConfig {
|
|
|
|
jCfg := &jsonConfig{}
|
|
|
|
|
|
|
|
if cfg.Folder != DefaultSubFolder {
|
|
|
|
jCfg.Folder = cfg.Folder
|
|
|
|
}
|
|
|
|
|
|
|
|
po := &pebbleOptions{}
|
|
|
|
po.Marshal(&cfg.PebbleOptions)
|
|
|
|
jCfg.PebbleOptions = *po
|
|
|
|
|
|
|
|
return jCfg
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetFolder returns the Pebble folder.
|
|
|
|
func (cfg *Config) GetFolder() string {
|
|
|
|
if filepath.IsAbs(cfg.Folder) {
|
|
|
|
return cfg.Folder
|
|
|
|
}
|
|
|
|
|
|
|
|
return filepath.Join(cfg.BaseDir, cfg.Folder)
|
|
|
|
}
|
|
|
|
|
|
|
|
// ToDisplayJSON returns JSON config as a string.
|
|
|
|
func (cfg *Config) ToDisplayJSON() ([]byte, error) {
|
|
|
|
return config.DisplayJSON(cfg.toJSONConfig())
|
|
|
|
}
|