ipfs-cluster/datastore/leveldb/config.go

package leveldb

import (
	"encoding/json"
	"errors"
	"path/filepath"

	"github.com/imdario/mergo"
	"github.com/ipfs/ipfs-cluster/config"
	"github.com/kelseyhightower/envconfig"
	goleveldb "github.com/syndtr/goleveldb/leveldb/opt"
)

const configKey = "leveldb"
const envConfigKey = "cluster_leveldb"

// Default values for LevelDB Config
const (
	DefaultSubFolder = "leveldb"
)

var (
	// DefaultLevelDBOptions carries default options. Values are customized during Init().
	DefaultLevelDBOptions goleveldb.Options
)

func init() {
	// go-ipfs uses defaults and only allows to configure compression, but
	// otherwise stores a small amount of values in LevelDB.
	// We leave defaults.
	// Example:
	DefaultLevelDBOptions.NoSync = false
}

// Config is used to initialize a LevelDB datastore. It implements the
// ComponentConfig interface.
type Config struct {
	config.Saver

	// The folder for this datastore. Non-absolute paths are relative to
	// the base configuration folder.
	Folder string

	LevelDBOptions goleveldb.Options
}

// levelDBOptions allows json serialization in our configuration of the
// goleveldb Options.
type levelDBOptions struct {
	BlockCacheCapacity                    int       `json:"block_cache_capacity"`
	BlockCacheEvictRemoved                bool      `json:"block_cache_evict_removed"`
	BlockRestartInterval                  int       `json:"block_restart_interval"`
	BlockSize                             int       `json:"block_size"`
	CompactionExpandLimitFactor           int       `json:"compaction_expand_limit_factor"`
	CompactionGPOverlapsFactor            int       `json:"compaction_gp_overlaps_factor"`
	CompactionL0Trigger                   int       `json:"compaction_l0_trigger"`
	CompactionSourceLimitFactor           int       `json:"compaction_source_limit_factor"`
	CompactionTableSize                   int       `json:"compaction_table_size"`
	CompactionTableSizeMultiplier         float64   `json:"compaction_table_size_multiplier"`
	CompactionTableSizeMultiplierPerLevel []float64 `json:"compaction_table_size_multiplier_per_level"`
	CompactionTotalSize                   int       `json:"compaction_total_size"`
	CompactionTotalSizeMultiplier         float64   `json:"compaction_total_size_multiplier"`
	CompactionTotalSizeMultiplierPerLevel []float64 `json:"compaction_total_size_multiplier_per_level"`
	Compression                           uint      `json:"compression"`
	DisableBufferPool                     bool      `json:"disable_buffer_pool"`
	DisableBlockCache                     bool      `json:"disable_block_cache"`
	DisableCompactionBackoff              bool      `json:"disable_compaction_backoff"`
	DisableLargeBatchTransaction          bool      `json:"disable_large_batch_transaction"`
	IteratorSamplingRate                  int       `json:"iterator_sampling_rate"`
	NoSync                                bool      `json:"no_sync"`
	NoWriteMerge                          bool      `json:"no_write_merge"`
	OpenFilesCacheCapacity                int       `json:"open_files_cache_capacity"`
	ReadOnly                              bool      `json:"read_only"`
	Strict                                uint      `json:"strict"`
	WriteBuffer                           int       `json:"write_buffer"`
	WriteL0PauseTrigger                   int       `json:"write_l0_pause_trigger"`
	WriteL0SlowdownTrigger                int       `json:"write_l0_slowdown_trigger"`
}

func (ldbo *levelDBOptions) Unmarshal() *goleveldb.Options {
	goldbo := &goleveldb.Options{}
	goldbo.BlockCacheCapacity = ldbo.BlockCacheCapacity
	goldbo.BlockCacheEvictRemoved = ldbo.BlockCacheEvictRemoved
	goldbo.BlockRestartInterval = ldbo.BlockRestartInterval
	goldbo.BlockSize = ldbo.BlockSize
	goldbo.CompactionExpandLimitFactor = ldbo.CompactionExpandLimitFactor
	goldbo.CompactionGPOverlapsFactor = ldbo.CompactionGPOverlapsFactor
	goldbo.CompactionL0Trigger = ldbo.CompactionL0Trigger
	goldbo.CompactionSourceLimitFactor = ldbo.CompactionSourceLimitFactor
	goldbo.CompactionTableSize = ldbo.CompactionTableSize
	goldbo.CompactionTableSizeMultiplier = ldbo.CompactionTableSizeMultiplier
	goldbo.CompactionTableSizeMultiplierPerLevel = ldbo.CompactionTableSizeMultiplierPerLevel
	goldbo.CompactionTotalSize = ldbo.CompactionTotalSize
	goldbo.CompactionTotalSizeMultiplier = ldbo.CompactionTotalSizeMultiplier
	goldbo.CompactionTotalSizeMultiplierPerLevel = ldbo.CompactionTotalSizeMultiplierPerLevel
	goldbo.Compression = goleveldb.Compression(ldbo.Compression)
	goldbo.DisableBufferPool = ldbo.DisableBufferPool
	goldbo.DisableBlockCache = ldbo.DisableBlockCache
	goldbo.DisableCompactionBackoff = ldbo.DisableCompactionBackoff
	goldbo.DisableLargeBatchTransaction = ldbo.DisableLargeBatchTransaction
	goldbo.IteratorSamplingRate = ldbo.IteratorSamplingRate
	goldbo.NoSync = ldbo.NoSync
	goldbo.NoWriteMerge = ldbo.NoWriteMerge
	goldbo.OpenFilesCacheCapacity = ldbo.OpenFilesCacheCapacity
	goldbo.ReadOnly = ldbo.ReadOnly
	goldbo.Strict = goleveldb.Strict(ldbo.Strict)
	goldbo.WriteBuffer = ldbo.WriteBuffer
	goldbo.WriteL0PauseTrigger = ldbo.WriteL0PauseTrigger
	goldbo.WriteL0SlowdownTrigger = ldbo.WriteL0SlowdownTrigger
	return goldbo
}

func (ldbo *levelDBOptions) Marshal(goldbo *goleveldb.Options) {
	ldbo.BlockCacheCapacity = goldbo.BlockCacheCapacity
	ldbo.BlockCacheEvictRemoved = goldbo.BlockCacheEvictRemoved
	ldbo.BlockRestartInterval = goldbo.BlockRestartInterval
	ldbo.BlockSize = goldbo.BlockSize
	ldbo.CompactionExpandLimitFactor = goldbo.CompactionExpandLimitFactor
	ldbo.CompactionGPOverlapsFactor = goldbo.CompactionGPOverlapsFactor
	ldbo.CompactionL0Trigger = goldbo.CompactionL0Trigger
	ldbo.CompactionSourceLimitFactor = goldbo.CompactionSourceLimitFactor
	ldbo.CompactionTableSize = goldbo.CompactionTableSize
	ldbo.CompactionTableSizeMultiplier = goldbo.CompactionTableSizeMultiplier
	ldbo.CompactionTableSizeMultiplierPerLevel = goldbo.CompactionTableSizeMultiplierPerLevel
	ldbo.CompactionTotalSize = goldbo.CompactionTotalSize
	ldbo.CompactionTotalSizeMultiplier = goldbo.CompactionTotalSizeMultiplier
	ldbo.CompactionTotalSizeMultiplierPerLevel = goldbo.CompactionTotalSizeMultiplierPerLevel
	ldbo.Compression = uint(goldbo.Compression)
	ldbo.DisableBufferPool = goldbo.DisableBufferPool
	ldbo.DisableBlockCache = goldbo.DisableBlockCache
	ldbo.DisableCompactionBackoff = goldbo.DisableCompactionBackoff
	ldbo.DisableLargeBatchTransaction = goldbo.DisableLargeBatchTransaction
	ldbo.IteratorSamplingRate = goldbo.IteratorSamplingRate
	ldbo.NoSync = goldbo.NoSync
	ldbo.NoWriteMerge = goldbo.NoWriteMerge
	ldbo.OpenFilesCacheCapacity = goldbo.OpenFilesCacheCapacity
	ldbo.ReadOnly = goldbo.ReadOnly
	ldbo.Strict = uint(goldbo.Strict)
	ldbo.WriteBuffer = goldbo.WriteBuffer
	ldbo.WriteL0PauseTrigger = goldbo.WriteL0PauseTrigger
	ldbo.WriteL0SlowdownTrigger = goldbo.WriteL0SlowdownTrigger
}

type jsonConfig struct {
	Folder         string         `json:"folder,omitempty"`
	LevelDBOptions levelDBOptions `json:"leveldb_options,omitempty"`
}

// ConfigKey returns a human-friendly identifier for this type of Datastore.
func (cfg *Config) ConfigKey() string {
	return configKey
}

// Default initializes this Config with sensible values.
func (cfg *Config) Default() error {
	cfg.Folder = DefaultSubFolder
	cfg.LevelDBOptions = DefaultLevelDBOptions
	return nil
}

// ApplyEnvVars fills in any Config fields found as environment variables.
func (cfg *Config) ApplyEnvVars() error {
	jcfg := cfg.toJSONConfig()

	err := envconfig.Process(envConfigKey, jcfg)
	if err != nil {
		return err
	}

	return cfg.applyJSONConfig(jcfg)
}

// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *Config) Validate() error {
	if cfg.Folder == "" {
		return errors.New("folder is unset")
	}

	return nil
}

// LoadJSON reads the fields of this Config from a JSON byteslice as
// generated by ToJSON.
func (cfg *Config) LoadJSON(raw []byte) error {
	jcfg := &jsonConfig{}
	err := json.Unmarshal(raw, jcfg)
	if err != nil {
		return err
	}
	cfg.Default()

	return cfg.applyJSONConfig(jcfg)
}

func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
	config.SetIfNotDefault(jcfg.Folder, &cfg.Folder)

	ldbOpts := jcfg.LevelDBOptions.Unmarshal()

	if err := mergo.Merge(&cfg.LevelDBOptions, ldbOpts, mergo.WithOverride); err != nil {
		return err
	}

	return cfg.Validate()
}

// ToJSON generates a JSON-formatted human-friendly representation of this
// Config.
func (cfg *Config) ToJSON() (raw []byte, err error) {
	jcfg := cfg.toJSONConfig()

	raw, err = config.DefaultJSONMarshal(jcfg)
	return
}

func (cfg *Config) toJSONConfig() *jsonConfig {
	jCfg := &jsonConfig{}

	if cfg.Folder != DefaultSubFolder {
		jCfg.Folder = cfg.Folder
	}

	bo := &levelDBOptions{}
	bo.Marshal(&cfg.LevelDBOptions)
	jCfg.LevelDBOptions = *bo

	return jCfg
}

// GetFolder returns the LevelDB folder.
func (cfg *Config) GetFolder() string {
	if filepath.IsAbs(cfg.Folder) {
		return cfg.Folder
	}

	return filepath.Join(cfg.BaseDir, cfg.Folder)
}

// ToDisplayJSON returns JSON config as a string.
func (cfg *Config) ToDisplayJSON() ([]byte, error) {
	return config.DisplayJSON(cfg.toJSONConfig())
}