expose badger loading mode conf

For simplicity, I haven't exposed the entire
badger config. Also, due to go default values and
the badger implementation, I have had to extract the
two loading mode config values so we can determine
whether it has actually has been set by the user in
the json config file.

License: MIT
Signed-off-by: Adrian Lanzafame <adrianlanzafame92@gmail.com>

datastore/badger/badger.go

@@ -18,8 +18,8 @@ func New(cfg *Config) (ds.Datastore, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "creating badger folder")
 	}
-	opts := &badgerds.DefaultOptions
+	opts := badgerds.Options{Options: cfg.BadgerOptions}
-	return badgerds.NewDatastore(folder, opts)
+	return badgerds.NewDatastore(folder, &opts)
 }
 
 // Cleanup deletes the badger datastore.

datastore/badger/config.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"path/filepath"
 
+	"github.com/dgraph-io/badger"
+	"github.com/dgraph-io/badger/options"
 	"github.com/kelseyhightower/envconfig"
 
 	"github.com/ipfs/ipfs-cluster/config"
@@ -18,6 +20,15 @@ const (
 	DefaultSubFolder = "badger"
 )
 
+var (
+	// DefaultBadgerOptions has to be a var because badger.DefaultOptions is.
+	DefaultBadgerOptions badger.Options
+)
+
+func init() {
+	DefaultBadgerOptions = badger.DefaultOptions
+}
+
 // Config is used to initialize a BadgerDB datastore. It implements the
 // ComponentConfig interface.
 type Config struct {
@@ -26,10 +37,29 @@ type Config struct {
 	// The folder for this datastore. Non-absolute paths are relative to
 	// the base configuration folder.
 	Folder string
+
+	BadgerOptions badger.Options
 }
 
+// FileLoadingMode specifies how data in LSM table files and value log files should
+// be loaded.
+type FileLoadingMode int
+
+const (
+	// Unknown indicates that the value hasn't been set
+	Unknown FileLoadingMode = iota
+	// FileIO indicates that files must be loaded using standard I/O
+	FileIO
+	// LoadToRAM indicates that file must be loaded into RAM
+	LoadToRAM
+	// MemoryMap indicates that that the file must be memory-mapped
+	MemoryMap
+)
+
 type jsonConfig struct {
-	Folder string `json:"folder,omitempty"`
+	Folder              string          `json:"folder,omitempty"`
+	TableLoadingMode    FileLoadingMode `json:"table_loading_mode,omitempty"`     // because we can't distinguish between options.FileIO and default value
+	ValueLogLoadingMode FileLoadingMode `json:"value_log_loading_mode,omitempty"` // because we can't distinguish between options.FileIO and default value
 }
 
 // ConfigKey returns a human-friendly identifier for this type of Datastore.
@@ -40,6 +70,7 @@ func (cfg *Config) ConfigKey() string {
 // Default initializes this Config with sensible values.
 func (cfg *Config) Default() error {
 	cfg.Folder = DefaultSubFolder
+	cfg.BadgerOptions = DefaultBadgerOptions
 	return nil
 }
 
@@ -81,6 +112,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	config.SetIfNotDefault(jcfg.Folder, &cfg.Folder)
+	if v, ok := correctLoadingMode(jcfg.TableLoadingMode); ok {
+		cfg.BadgerOptions.TableLoadingMode = options.FileLoadingMode(v)
+	}
+	if v, ok := correctLoadingMode(jcfg.ValueLogLoadingMode); ok {
+		cfg.BadgerOptions.ValueLogLoadingMode = options.FileLoadingMode(v)
+	}
 	return cfg.Validate()
 }
 
@@ -98,6 +135,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig {
 
 	if cfg.Folder != DefaultSubFolder {
 		jCfg.Folder = cfg.Folder
+		jCfg.TableLoadingMode = FileLoadingMode(cfg.BadgerOptions.TableLoadingMode)
+		jCfg.ValueLogLoadingMode = FileLoadingMode(cfg.BadgerOptions.ValueLogLoadingMode)
 	}
 
 	return jCfg
@@ -111,3 +150,12 @@ func (cfg *Config) GetFolder() string {
 
 	return filepath.Join(cfg.BaseDir, cfg.Folder)
 }
+
+func correctLoadingMode(lm FileLoadingMode) (options.FileLoadingMode, bool) {
+	switch lm {
+	default:
+		return 0, false
+	case 1, 2, 3:
+		return options.FileLoadingMode(lm - 1), true
+	}
+}

datastore/badger/config_test.go

@@ -0,0 +1,53 @@
+package badger
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/dgraph-io/badger"
+	"github.com/dgraph-io/badger/options"
+)
+
+var cfgJSON = []byte(`
+{
+    "folder": "test",
+	"value_log_loading_mode": 1
+}
+`)
+
+func TestLoadJSON(t *testing.T) {
+	cfg := &Config{}
+	err := cfg.LoadJSON(cfgJSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestToJSON(t *testing.T) {
+	cfg := &Config{}
+	cfg.LoadJSON(cfgJSON)
+
+	if cfg.BadgerOptions.ValueLogLoadingMode != options.FileIO {
+		t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.ValueLogLoadingMode, options.FileIO)
+	}
+	if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions.ValueLogFileSize {
+		t.Fatalf(
+			"got: %d, want: %d",
+			cfg.BadgerOptions.ValueLogFileSize,
+			badger.DefaultOptions.ValueLogFileSize,
+		)
+	}
+
+	fmt.Printf("%+v\n", cfg.BadgerOptions)
+
+	newjson, err := cfg.ToJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfg = &Config{}
+	err = cfg.LoadJSON(newjson)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

go.mod

@@ -6,6 +6,10 @@ require (
 	github.com/ajstarks/svgo v0.0.0-20181006003313-6ce6a3bcf6cd // indirect
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/btcsuite/btcd v0.0.0-20190427004231-96897255fd17 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/dgraph-io/badger v2.0.0-rc.2+incompatible
+	github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fogleman/gg v1.3.0 // indirect
 	github.com/gogo/protobuf v1.2.1

go.sum

@@ -81,6 +81,8 @@ github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc h1:VxEJYcOh1LMAdhIiHkofa6UC0PZvCmielUgJXgAAWFU=
+github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -584,6 +586,10 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+<<<<<<< HEAD
+=======
+github.com/ugorji/go v0.0.0-20171015030454-f26fc641ec9d/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
+>>>>>>> expose badger loading mode conf
 github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=