expose badger loading mode conf
For simplicity, I haven't exposed the entire badger config. Also, due to go default values and the badger implementation, I have had to extract the two loading mode config values so we can determine whether it has actually has been set by the user in the json config file. License: MIT Signed-off-by: Adrian Lanzafame <adrianlanzafame92@gmail.com>
This commit is contained in:
parent
e523215ee2
commit
3d4412ffd4
|
@ -18,8 +18,8 @@ func New(cfg *Config) (ds.Datastore, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "creating badger folder")
|
return nil, errors.Wrap(err, "creating badger folder")
|
||||||
}
|
}
|
||||||
opts := &badgerds.DefaultOptions
|
opts := badgerds.Options{Options: cfg.BadgerOptions}
|
||||||
return badgerds.NewDatastore(folder, opts)
|
return badgerds.NewDatastore(folder, &opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleanup deletes the badger datastore.
|
// Cleanup deletes the badger datastore.
|
||||||
|
|
|
@ -5,6 +5,8 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/dgraph-io/badger"
|
||||||
|
"github.com/dgraph-io/badger/options"
|
||||||
"github.com/kelseyhightower/envconfig"
|
"github.com/kelseyhightower/envconfig"
|
||||||
|
|
||||||
"github.com/ipfs/ipfs-cluster/config"
|
"github.com/ipfs/ipfs-cluster/config"
|
||||||
|
@ -18,6 +20,15 @@ const (
|
||||||
DefaultSubFolder = "badger"
|
DefaultSubFolder = "badger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// DefaultBadgerOptions has to be a var because badger.DefaultOptions is.
|
||||||
|
DefaultBadgerOptions badger.Options
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
DefaultBadgerOptions = badger.DefaultOptions
|
||||||
|
}
|
||||||
|
|
||||||
// Config is used to initialize a BadgerDB datastore. It implements the
|
// Config is used to initialize a BadgerDB datastore. It implements the
|
||||||
// ComponentConfig interface.
|
// ComponentConfig interface.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
|
@ -26,10 +37,29 @@ type Config struct {
|
||||||
// The folder for this datastore. Non-absolute paths are relative to
|
// The folder for this datastore. Non-absolute paths are relative to
|
||||||
// the base configuration folder.
|
// the base configuration folder.
|
||||||
Folder string
|
Folder string
|
||||||
|
|
||||||
|
BadgerOptions badger.Options
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FileLoadingMode specifies how data in LSM table files and value log files should
|
||||||
|
// be loaded.
|
||||||
|
type FileLoadingMode int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// Unknown indicates that the value hasn't been set
|
||||||
|
Unknown FileLoadingMode = iota
|
||||||
|
// FileIO indicates that files must be loaded using standard I/O
|
||||||
|
FileIO
|
||||||
|
// LoadToRAM indicates that file must be loaded into RAM
|
||||||
|
LoadToRAM
|
||||||
|
// MemoryMap indicates that that the file must be memory-mapped
|
||||||
|
MemoryMap
|
||||||
|
)
|
||||||
|
|
||||||
type jsonConfig struct {
|
type jsonConfig struct {
|
||||||
Folder string `json:"folder,omitempty"`
|
Folder string `json:"folder,omitempty"`
|
||||||
|
TableLoadingMode FileLoadingMode `json:"table_loading_mode,omitempty"` // because we can't distinguish between options.FileIO and default value
|
||||||
|
ValueLogLoadingMode FileLoadingMode `json:"value_log_loading_mode,omitempty"` // because we can't distinguish between options.FileIO and default value
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConfigKey returns a human-friendly identifier for this type of Datastore.
|
// ConfigKey returns a human-friendly identifier for this type of Datastore.
|
||||||
|
@ -40,6 +70,7 @@ func (cfg *Config) ConfigKey() string {
|
||||||
// Default initializes this Config with sensible values.
|
// Default initializes this Config with sensible values.
|
||||||
func (cfg *Config) Default() error {
|
func (cfg *Config) Default() error {
|
||||||
cfg.Folder = DefaultSubFolder
|
cfg.Folder = DefaultSubFolder
|
||||||
|
cfg.BadgerOptions = DefaultBadgerOptions
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -81,6 +112,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
|
||||||
|
|
||||||
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
|
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
|
||||||
config.SetIfNotDefault(jcfg.Folder, &cfg.Folder)
|
config.SetIfNotDefault(jcfg.Folder, &cfg.Folder)
|
||||||
|
if v, ok := correctLoadingMode(jcfg.TableLoadingMode); ok {
|
||||||
|
cfg.BadgerOptions.TableLoadingMode = options.FileLoadingMode(v)
|
||||||
|
}
|
||||||
|
if v, ok := correctLoadingMode(jcfg.ValueLogLoadingMode); ok {
|
||||||
|
cfg.BadgerOptions.ValueLogLoadingMode = options.FileLoadingMode(v)
|
||||||
|
}
|
||||||
return cfg.Validate()
|
return cfg.Validate()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -98,6 +135,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig {
|
||||||
|
|
||||||
if cfg.Folder != DefaultSubFolder {
|
if cfg.Folder != DefaultSubFolder {
|
||||||
jCfg.Folder = cfg.Folder
|
jCfg.Folder = cfg.Folder
|
||||||
|
jCfg.TableLoadingMode = FileLoadingMode(cfg.BadgerOptions.TableLoadingMode)
|
||||||
|
jCfg.ValueLogLoadingMode = FileLoadingMode(cfg.BadgerOptions.ValueLogLoadingMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
return jCfg
|
return jCfg
|
||||||
|
@ -111,3 +150,12 @@ func (cfg *Config) GetFolder() string {
|
||||||
|
|
||||||
return filepath.Join(cfg.BaseDir, cfg.Folder)
|
return filepath.Join(cfg.BaseDir, cfg.Folder)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func correctLoadingMode(lm FileLoadingMode) (options.FileLoadingMode, bool) {
|
||||||
|
switch lm {
|
||||||
|
default:
|
||||||
|
return 0, false
|
||||||
|
case 1, 2, 3:
|
||||||
|
return options.FileLoadingMode(lm - 1), true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
53
datastore/badger/config_test.go
Normal file
53
datastore/badger/config_test.go
Normal file
|
@ -0,0 +1,53 @@
|
||||||
|
package badger
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/dgraph-io/badger"
|
||||||
|
"github.com/dgraph-io/badger/options"
|
||||||
|
)
|
||||||
|
|
||||||
|
var cfgJSON = []byte(`
|
||||||
|
{
|
||||||
|
"folder": "test",
|
||||||
|
"value_log_loading_mode": 1
|
||||||
|
}
|
||||||
|
`)
|
||||||
|
|
||||||
|
func TestLoadJSON(t *testing.T) {
|
||||||
|
cfg := &Config{}
|
||||||
|
err := cfg.LoadJSON(cfgJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestToJSON(t *testing.T) {
|
||||||
|
cfg := &Config{}
|
||||||
|
cfg.LoadJSON(cfgJSON)
|
||||||
|
|
||||||
|
if cfg.BadgerOptions.ValueLogLoadingMode != options.FileIO {
|
||||||
|
t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.ValueLogLoadingMode, options.FileIO)
|
||||||
|
}
|
||||||
|
if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions.ValueLogFileSize {
|
||||||
|
t.Fatalf(
|
||||||
|
"got: %d, want: %d",
|
||||||
|
cfg.BadgerOptions.ValueLogFileSize,
|
||||||
|
badger.DefaultOptions.ValueLogFileSize,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("%+v\n", cfg.BadgerOptions)
|
||||||
|
|
||||||
|
newjson, err := cfg.ToJSON()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg = &Config{}
|
||||||
|
err = cfg.LoadJSON(newjson)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
4
go.mod
4
go.mod
|
@ -6,6 +6,10 @@ require (
|
||||||
github.com/ajstarks/svgo v0.0.0-20181006003313-6ce6a3bcf6cd // indirect
|
github.com/ajstarks/svgo v0.0.0-20181006003313-6ce6a3bcf6cd // indirect
|
||||||
github.com/blang/semver v3.5.1+incompatible
|
github.com/blang/semver v3.5.1+incompatible
|
||||||
github.com/boltdb/bolt v1.3.1 // indirect
|
github.com/boltdb/bolt v1.3.1 // indirect
|
||||||
|
github.com/btcsuite/btcd v0.0.0-20190427004231-96897255fd17 // indirect
|
||||||
|
github.com/coreos/go-semver v0.3.0 // indirect
|
||||||
|
github.com/dgraph-io/badger v2.0.0-rc.2+incompatible
|
||||||
|
github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc // indirect
|
||||||
github.com/dustin/go-humanize v1.0.0
|
github.com/dustin/go-humanize v1.0.0
|
||||||
github.com/fogleman/gg v1.3.0 // indirect
|
github.com/fogleman/gg v1.3.0 // indirect
|
||||||
github.com/gogo/protobuf v1.2.1
|
github.com/gogo/protobuf v1.2.1
|
||||||
|
|
6
go.sum
6
go.sum
|
@ -81,6 +81,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl
|
||||||
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
||||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||||
|
github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc h1:VxEJYcOh1LMAdhIiHkofa6UC0PZvCmielUgJXgAAWFU=
|
||||||
|
github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
||||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||||
|
@ -584,6 +586,10 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
|
||||||
|
<<<<<<< HEAD
|
||||||
|
=======
|
||||||
|
github.com/ugorji/go v0.0.0-20171015030454-f26fc641ec9d/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
|
||||||
|
>>>>>>> expose badger loading mode conf
|
||||||
github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
|
github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
|
||||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||||
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
|
github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
|
||||||
|
|
Loading…
Reference in New Issue
Block a user