From 970bdc8cb0f20035edcb812e36f54d0dcf80388a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 29 Nov 2022 17:09:19 +0100 Subject: [PATCH] Add Badger v3 support This commit adds badger v3 support as datastore backend for IPFS Cluster. Badgerv3 has a number of improvements over v1, particularly around compaction. It is significantly more modern and should behave better in general (some testing is pending). --- cmd/ipfs-cluster-service/main.go | 6 +- cmdutils/configs.go | 11 +- datastore/badger/badger.go | 3 + datastore/badger/config.go | 1 + datastore/badger3/badger.go | 40 ++++ datastore/badger3/config.go | 360 +++++++++++++++++++++++++++++++ datastore/badger3/config_test.go | 90 ++++++++ go.mod | 10 +- go.sum | 16 ++ logging.go | 1 + 10 files changed, 531 insertions(+), 7 deletions(-) create mode 100644 datastore/badger3/badger.go create mode 100644 datastore/badger3/config.go create mode 100644 datastore/badger3/config_test.go diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 1a4bb1d8..d229f465 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -269,12 +269,12 @@ the peer IDs in the given multiaddresses. Flags: []cli.Flag{ cli.StringFlag{ Name: "consensus", - Usage: "select consensus component: 'crdt' or 'raft'", + Usage: "select consensus: 'crdt' or 'raft'", Value: defaultConsensus, }, cli.StringFlag{ Name: "datastore", - Usage: "select datastore component: 'badger' or 'leveldb'", + Usage: "select datastore: 'badger', 'badger3' or 'leveldb'", Value: defaultDatastore, }, cli.BoolFlag{ @@ -304,7 +304,7 @@ the peer IDs in the given multiaddresses. datastore := c.String("datastore") switch datastore { - case "leveldb", "badger": + case "leveldb", "badger", "badger3": default: checkErr("choosing datastore", errors.New("flag value must be set to 'leveldb' or 'badger'")) } diff --git a/cmdutils/configs.go b/cmdutils/configs.go index 46c02dfa..f658bdc0 100644 --- a/cmdutils/configs.go +++ b/cmdutils/configs.go @@ -16,6 +16,7 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/informer/numpin" @@ -46,6 +47,7 @@ type Configs struct { Metrics *observations.MetricsConfig Tracing *observations.TracingConfig Badger *badger.Config + Badger3 *badger3.Config LevelDB *leveldb.Config } @@ -195,10 +197,11 @@ func (ch *ConfigHelper) GetDatastore() string { } badgerLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Badger.ConfigKey()) + badger3Loaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Badger3.ConfigKey()) levelDBLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.LevelDB.ConfigKey()) nLoaded := 0 - for _, v := range []bool{badgerLoaded, levelDBLoaded} { + for _, v := range []bool{badgerLoaded, badger3Loaded, levelDBLoaded} { if v { nLoaded++ } @@ -209,6 +212,8 @@ func (ch *ConfigHelper) GetDatastore() string { switch { case badgerLoaded: return ch.configs.Badger.ConfigKey() + case badger3Loaded: + return ch.configs.Badger3.ConfigKey() case levelDBLoaded: return ch.configs.LevelDB.ConfigKey() default: @@ -237,6 +242,7 @@ func (ch *ConfigHelper) init() { Metrics: &observations.MetricsConfig{}, Tracing: &observations.TracingConfig{}, Badger: &badger.Config{}, + Badger3: &badger3.Config{}, LevelDB: &leveldb.Config{}, } man.RegisterComponent(config.Cluster, cfgs.Cluster) @@ -272,12 +278,15 @@ func (ch *ConfigHelper) init() { switch ch.datastore { case cfgs.Badger.ConfigKey(): man.RegisterComponent(config.Datastore, cfgs.Badger) + case cfgs.Badger3.ConfigKey(): + man.RegisterComponent(config.Datastore, cfgs.Badger3) case cfgs.LevelDB.ConfigKey(): man.RegisterComponent(config.Datastore, cfgs.LevelDB) default: man.RegisterComponent(config.Datastore, cfgs.LevelDB) man.RegisterComponent(config.Datastore, cfgs.Badger) + man.RegisterComponent(config.Datastore, cfgs.Badger3) } } diff --git a/datastore/badger/badger.go b/datastore/badger/badger.go index 2a2fabd3..26e0e90d 100644 --- a/datastore/badger/badger.go +++ b/datastore/badger/badger.go @@ -7,9 +7,12 @@ import ( ds "github.com/ipfs/go-datastore" badgerds "github.com/ipfs/go-ds-badger" + logging "github.com/ipfs/go-log/v2" "github.com/pkg/errors" ) +var logger = logging.Logger("badger") + // New returns a BadgerDB datastore configured with the given // configuration. func New(cfg *Config) (ds.Datastore, error) { diff --git a/datastore/badger/config.go b/datastore/badger/config.go index 1b57531b..2f6ede0d 100644 --- a/datastore/badger/config.go +++ b/datastore/badger/config.go @@ -169,6 +169,7 @@ func (cfg *Config) Default() error { cfg.GCInterval = DefaultGCInterval cfg.GCSleep = DefaultGCSleep cfg.BadgerOptions = DefaultBadgerOptions + cfg.BadgerOptions.Logger = logger return nil } diff --git a/datastore/badger3/badger.go b/datastore/badger3/badger.go new file mode 100644 index 00000000..04f8cdf3 --- /dev/null +++ b/datastore/badger3/badger.go @@ -0,0 +1,40 @@ +// Package badger3 provides a configurable BadgerDB v3 go-datastore for use with +// IPFS Cluster. +package badger3 + +import ( + "os" + + ds "github.com/ipfs/go-datastore" + badgerds "github.com/ipfs/go-ds-badger3" + logging "github.com/ipfs/go-log/v2" + "github.com/pkg/errors" +) + +var logger = logging.Logger("badger3") + +// New returns a BadgerDB datastore configured with the given +// configuration. +func New(cfg *Config) (ds.Datastore, error) { + folder := cfg.GetFolder() + err := os.MkdirAll(folder, 0700) + if err != nil { + return nil, errors.Wrap(err, "creating badger folder") + } + opts := badgerds.Options{ + GcDiscardRatio: cfg.GCDiscardRatio, + GcInterval: cfg.GCInterval, + GcSleep: cfg.GCSleep, + Options: cfg.BadgerOptions, + } + return badgerds.NewDatastore(folder, &opts) +} + +// Cleanup deletes the badger datastore. +func Cleanup(cfg *Config) error { + folder := cfg.GetFolder() + if _, err := os.Stat(folder); os.IsNotExist(err) { + return nil + } + return os.RemoveAll(cfg.GetFolder()) +} diff --git a/datastore/badger3/config.go b/datastore/badger3/config.go new file mode 100644 index 00000000..adf28d4e --- /dev/null +++ b/datastore/badger3/config.go @@ -0,0 +1,360 @@ +package badger3 + +import ( + "encoding/json" + "errors" + "path/filepath" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v3/options" + "github.com/imdario/mergo" + "github.com/kelseyhightower/envconfig" + + "github.com/ipfs-cluster/ipfs-cluster/config" +) + +const configKey = "badger3" +const envConfigKey = "cluster_badger3" + +// Default values for badger Config +const ( + DefaultSubFolder = "badger3" +) + +var ( + // DefaultBadgerOptions has to be a var because badger.DefaultOptions + // is. Values are customized during Init(). + DefaultBadgerOptions badger.Options + + // DefaultGCDiscardRatio for GC operations. See Badger docs. + DefaultGCDiscardRatio float64 = 0.2 + // DefaultGCInterval specifies interval between GC cycles. + DefaultGCInterval time.Duration = 15 * time.Minute + // DefaultGCSleep specifies sleep time between GC rounds. + DefaultGCSleep time.Duration = 10 * time.Second +) + +func init() { + DefaultBadgerOptions = badger.DefaultOptions("") + // Better to slow down starts than shutdowns. + DefaultBadgerOptions.CompactL0OnClose = false + // Defaults to 1MB! For us that means everything goes into the LSM + // tree and the LSM tree is supposed to be loaded into memory in full. + // We only put very small things on the LSM tree by default (i.e. a + // single CID). + DefaultBadgerOptions.ValueThreshold = 100 + // Disable Block Cache: the cluster read-pattern at scale requires + // looping regularly all keys. The CRDT read-patterm avoids reading + // something twice. In general, it probably does not add much, and it + // is recommended to be disabled when not using compression. + DefaultBadgerOptions.BlockCacheSize = 0 + // Let's disable compression for values, better perf when reading and + // usually the ratio between data stored by badger and the cluster + // should be small. Users can always enable. + DefaultBadgerOptions.Compression = options.None + // There is a write lock in go-ds-crdt that writes batches one by one. + // Also NewWriteBatch says that there can never be transaction + // conflicts when doing batches. And IPFS will only write a block + // once, or do it with the same values. In general, we probably don't + // care about conflicts much (rows updated while a commit transaction + // was open). Increases perf too. + DefaultBadgerOptions.DetectConflicts = false + // TODO: Increase memtable size. This will use some more memory, but any + // normal system should be able to deal with using 256MiB for the + // memtable. Badger puts a lot of things in memory anyways, + // i.e. IndexCacheSize is set to 0. Note NumMemTables is 5. + // DefaultBadgerOptions.MemTableSize = 268435456 // 256MiB + +} + +// Config is used to initialize a BadgerDB datastore. It implements the +// ComponentConfig interface. +type Config struct { + config.Saver + + // The folder for this datastore. Non-absolute paths are relative to + // the base configuration folder. + Folder string + + // For GC operation. See Badger documentation. + GCDiscardRatio float64 + + // Interval between GC cycles. Each GC cycle runs one or more + // rounds separated by GCSleep. + GCInterval time.Duration + + // Time between rounds in a GC cycle + GCSleep time.Duration + + BadgerOptions badger.Options +} + +// badgerOptions is a copy of badger.Options so it can be marshaled by us. +type badgerOptions struct { + Dir string `json:"dir"` + ValueDir string `json:"value_dir"` + SyncWrites bool `json:"sync_writes"` + NumVersionsToKeep int `json:"num_versions_to_keep"` + ReadOnly bool `json:"read_only"` + // Logger + Compression options.CompressionType `json:"compression"` + InMemory bool `json:"in_memory"` + MetricsEnabled bool `json:"metrics_enabled"` + NumGoroutines int `json:"num_goroutines"` + + MemTableSize int64 `json:"mem_table_size"` + BaseTableSize int64 `json:"base_table_size"` + BaseLevelSize int64 `json:"base_level_size"` + LevelSizeMultiplier int `json:"level_size_multiplier"` + TableSizeMultiplier int `json:"table_size_multiplier"` + MaxLevels int `json:"max_levels"` + + VLogPercentile float64 `json:"v_log_percentile"` + ValueThreshold int64 `json:"value_threshold"` + NumMemtables int `json:"num_memtables"` + BlockSize int `json:"block_size"` + BloomFalsePositive float64 `json:"bloom_false_positive"` + BlockCacheSize int64 `json:"block_cache_size"` + IndexCacheSize int64 `json:"index_cache_size"` + + NumLevelZeroTables int `json:"num_level_zero_tables"` + NumLevelZeroTablesStall int `json:"num_level_zero_tables_stall"` + + ValueLogFileSize int64 `json:"value_log_file_size"` + ValueLogMaxEntries uint32 `json:"value_log_max_entries"` + + NumCompactors int `json:"num_compactors"` + CompactL0OnClose bool `json:"compact_l_0_on_close"` + LmaxCompaction bool `json:"lmax_compaction"` + ZSTDCompressionLevel int `json:"zstd_compression_level"` + + VerifyValueChecksum bool `json:"verify_value_checksum"` + + ChecksumVerificationMode options.ChecksumVerificationMode `json:"checksum_verification_mode"` + DetectConflicts bool `json:"detect_conflicts"` + + NamespaceOffset int `json:"namespace_offset"` +} + +func (bo *badgerOptions) Unmarshal() *badger.Options { + badgerOpts := &badger.Options{} + badgerOpts.Dir = bo.Dir + badgerOpts.ValueDir = bo.ValueDir + badgerOpts.SyncWrites = bo.SyncWrites + badgerOpts.NumVersionsToKeep = bo.NumVersionsToKeep + badgerOpts.ReadOnly = bo.ReadOnly + badgerOpts.Compression = bo.Compression + badgerOpts.InMemory = bo.InMemory + badgerOpts.MetricsEnabled = bo.MetricsEnabled + badgerOpts.NumGoroutines = bo.NumGoroutines + + badgerOpts.MemTableSize = bo.MemTableSize + badgerOpts.BaseTableSize = bo.BaseTableSize + badgerOpts.BaseLevelSize = bo.BaseLevelSize + badgerOpts.LevelSizeMultiplier = bo.LevelSizeMultiplier + badgerOpts.TableSizeMultiplier = bo.TableSizeMultiplier + badgerOpts.MaxLevels = bo.MaxLevels + + badgerOpts.VLogPercentile = bo.VLogPercentile + badgerOpts.ValueThreshold = bo.ValueThreshold + badgerOpts.NumMemtables = bo.NumMemtables + badgerOpts.BlockSize = bo.BlockSize + badgerOpts.BloomFalsePositive = bo.BloomFalsePositive + badgerOpts.BlockCacheSize = bo.BlockCacheSize + badgerOpts.IndexCacheSize = bo.IndexCacheSize + + badgerOpts.NumLevelZeroTables = bo.NumLevelZeroTables + badgerOpts.NumLevelZeroTablesStall = bo.NumLevelZeroTablesStall + + badgerOpts.ValueLogFileSize = bo.ValueLogFileSize + badgerOpts.ValueLogMaxEntries = bo.ValueLogMaxEntries + + badgerOpts.NumCompactors = bo.NumCompactors + badgerOpts.CompactL0OnClose = bo.CompactL0OnClose + badgerOpts.LmaxCompaction = bo.LmaxCompaction + badgerOpts.ZSTDCompressionLevel = bo.ZSTDCompressionLevel + + badgerOpts.VerifyValueChecksum = bo.VerifyValueChecksum + + badgerOpts.ChecksumVerificationMode = bo.ChecksumVerificationMode + badgerOpts.DetectConflicts = bo.DetectConflicts + + badgerOpts.NamespaceOffset = bo.NamespaceOffset + + return badgerOpts +} + +func (bo *badgerOptions) Marshal(badgerOpts *badger.Options) { + bo.Dir = badgerOpts.Dir + bo.ValueDir = badgerOpts.ValueDir + bo.SyncWrites = badgerOpts.SyncWrites + bo.NumVersionsToKeep = badgerOpts.NumVersionsToKeep + bo.ReadOnly = badgerOpts.ReadOnly + bo.Compression = badgerOpts.Compression + bo.InMemory = badgerOpts.InMemory + bo.MetricsEnabled = badgerOpts.MetricsEnabled + bo.NumGoroutines = badgerOpts.NumGoroutines + + bo.MemTableSize = badgerOpts.MemTableSize + bo.BaseTableSize = badgerOpts.BaseTableSize + bo.BaseLevelSize = badgerOpts.BaseLevelSize + bo.LevelSizeMultiplier = badgerOpts.LevelSizeMultiplier + bo.TableSizeMultiplier = badgerOpts.TableSizeMultiplier + bo.MaxLevels = badgerOpts.MaxLevels + + bo.VLogPercentile = badgerOpts.VLogPercentile + bo.ValueThreshold = badgerOpts.ValueThreshold + bo.NumMemtables = badgerOpts.NumMemtables + bo.BlockSize = badgerOpts.BlockSize + bo.BloomFalsePositive = badgerOpts.BloomFalsePositive + bo.BlockCacheSize = badgerOpts.BlockCacheSize + bo.IndexCacheSize = badgerOpts.IndexCacheSize + + bo.NumLevelZeroTables = badgerOpts.NumLevelZeroTables + bo.NumLevelZeroTablesStall = badgerOpts.NumLevelZeroTablesStall + + bo.ValueLogFileSize = badgerOpts.ValueLogFileSize + bo.ValueLogMaxEntries = badgerOpts.ValueLogMaxEntries + + bo.NumCompactors = badgerOpts.NumCompactors + bo.CompactL0OnClose = badgerOpts.CompactL0OnClose + bo.LmaxCompaction = badgerOpts.LmaxCompaction + bo.ZSTDCompressionLevel = badgerOpts.ZSTDCompressionLevel + + bo.VerifyValueChecksum = badgerOpts.VerifyValueChecksum + + bo.ChecksumVerificationMode = badgerOpts.ChecksumVerificationMode + bo.DetectConflicts = badgerOpts.DetectConflicts + + bo.NamespaceOffset = badgerOpts.NamespaceOffset +} + +type jsonConfig struct { + Folder string `json:"folder,omitempty"` + GCDiscardRatio float64 `json:"gc_discard_ratio"` + GCInterval string `json:"gc_interval"` + GCSleep string `json:"gc_sleep"` + BadgerOptions badgerOptions `json:"badger_options,omitempty"` +} + +// ConfigKey returns a human-friendly identifier for this type of Datastore. +func (cfg *Config) ConfigKey() string { + return configKey +} + +// Default initializes this Config with sensible values. +func (cfg *Config) Default() error { + cfg.Folder = DefaultSubFolder + cfg.GCDiscardRatio = DefaultGCDiscardRatio + cfg.GCInterval = DefaultGCInterval + cfg.GCSleep = DefaultGCSleep + cfg.BadgerOptions = DefaultBadgerOptions + cfg.BadgerOptions.Logger = logger + return nil +} + +// ApplyEnvVars fills in any Config fields found as environment variables. +func (cfg *Config) ApplyEnvVars() error { + jcfg := cfg.toJSONConfig() + + err := envconfig.Process(envConfigKey, jcfg) + if err != nil { + return err + } + + return cfg.applyJSONConfig(jcfg) +} + +// Validate checks that the fields of this Config have working values, +// at least in appearance. +func (cfg *Config) Validate() error { + if cfg.Folder == "" { + return errors.New("folder is unset") + } + + if cfg.GCDiscardRatio <= 0 || cfg.GCDiscardRatio >= 1 { + return errors.New("gc_discard_ratio must be more than 0 and less than 1") + } + + return nil +} + +// LoadJSON reads the fields of this Config from a JSON byteslice as +// generated by ToJSON. +func (cfg *Config) LoadJSON(raw []byte) error { + jcfg := &jsonConfig{} + err := json.Unmarshal(raw, jcfg) + if err != nil { + return err + } + cfg.Default() + + return cfg.applyJSONConfig(jcfg) +} + +func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { + config.SetIfNotDefault(jcfg.Folder, &cfg.Folder) + + // 0 is an invalid option anyways. In that case, set default (0.2) + config.SetIfNotDefault(jcfg.GCDiscardRatio, &cfg.GCDiscardRatio) + + // If these durations are set, GC is enabled by default with default + // values. + err := config.ParseDurations("badger", + &config.DurationOpt{Duration: jcfg.GCInterval, Dst: &cfg.GCInterval, Name: "gc_interval"}, + &config.DurationOpt{Duration: jcfg.GCSleep, Dst: &cfg.GCSleep, Name: "gc_sleep"}, + ) + if err != nil { + return err + } + + badgerOpts := jcfg.BadgerOptions.Unmarshal() + + if err := mergo.Merge(&cfg.BadgerOptions, badgerOpts, mergo.WithOverride); err != nil { + return err + } + + return cfg.Validate() +} + +// ToJSON generates a JSON-formatted human-friendly representation of this +// Config. +func (cfg *Config) ToJSON() (raw []byte, err error) { + jcfg := cfg.toJSONConfig() + + raw, err = config.DefaultJSONMarshal(jcfg) + return +} + +func (cfg *Config) toJSONConfig() *jsonConfig { + jCfg := &jsonConfig{} + + if cfg.Folder != DefaultSubFolder { + jCfg.Folder = cfg.Folder + } + + jCfg.GCDiscardRatio = cfg.GCDiscardRatio + jCfg.GCInterval = cfg.GCInterval.String() + jCfg.GCSleep = cfg.GCSleep.String() + + bo := &badgerOptions{} + bo.Marshal(&cfg.BadgerOptions) + jCfg.BadgerOptions = *bo + + return jCfg +} + +// GetFolder returns the BadgerDB folder. +func (cfg *Config) GetFolder() string { + if filepath.IsAbs(cfg.Folder) { + return cfg.Folder + } + + return filepath.Join(cfg.BaseDir, cfg.Folder) +} + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/datastore/badger3/config_test.go b/datastore/badger3/config_test.go new file mode 100644 index 00000000..a2110d71 --- /dev/null +++ b/datastore/badger3/config_test.go @@ -0,0 +1,90 @@ +package badger3 + +import ( + "testing" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v3/options" +) + +var cfgJSON = []byte(` +{ + "folder": "test", + "gc_discard_ratio": 0.1, + "gc_sleep": "2m", + "badger_options": { + "max_levels": 4, + "compression": 2 + } +} +`) + +func TestLoadJSON(t *testing.T) { + cfg := &Config{} + err := cfg.LoadJSON(cfgJSON) + if err != nil { + t.Fatal(err) + } +} + +func TestToJSON(t *testing.T) { + cfg := &Config{} + cfg.LoadJSON(cfgJSON) + + if cfg.GCDiscardRatio != 0.1 { + t.Fatal("GCDiscardRatio should be 0.1") + } + + if cfg.GCInterval != DefaultGCInterval { + t.Fatal("GCInterval should default as it is unset") + } + + if cfg.GCSleep != 2*time.Minute { + t.Fatal("GCSleep should be 2m") + } + + if cfg.BadgerOptions.Compression != options.ZSTD { + t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.Compression, options.ZSTD) + } + + if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions("").ValueLogFileSize { + t.Fatalf( + "got: %d, want: %d", + cfg.BadgerOptions.ValueLogFileSize, + badger.DefaultOptions("").ValueLogFileSize, + ) + } + + if cfg.BadgerOptions.ChecksumVerificationMode != badger.DefaultOptions("").ChecksumVerificationMode { + t.Fatalf("ChecksumVerificationMode is not nil: got: %v, want: %v", cfg.BadgerOptions.ChecksumVerificationMode, badger.DefaultOptions("").ChecksumVerificationMode) + } + + if cfg.BadgerOptions.MaxLevels != 4 { + t.Fatalf("MaxLevels should be 4, got: %d", cfg.BadgerOptions.MaxLevels) + } + + newjson, err := cfg.ToJSON() + if err != nil { + t.Fatal(err) + } + + cfg = &Config{} + err = cfg.LoadJSON(newjson) + if err != nil { + t.Fatal(err) + } +} + +func TestDefault(t *testing.T) { + cfg := &Config{} + cfg.Default() + if cfg.Validate() != nil { + t.Fatal("error validating") + } + + cfg.GCDiscardRatio = 0 + if cfg.Validate() == nil { + t.Fatal("expected error validating") + } +} diff --git a/go.mod b/go.mod index 1b45fc36..11aba051 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,8 @@ require ( github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/dgraph-io/ristretto v0.0.2 // indirect + github.com/dgraph-io/badger/v3 v3.2103.4 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/fatih/color v1.13.0 // indirect @@ -100,9 +101,11 @@ require ( github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -117,6 +120,7 @@ require ( github.com/ipfs/go-bitswap v0.10.0 // indirect github.com/ipfs/go-blockservice v0.4.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect + github.com/ipfs/go-ds-badger3 v0.0.2 // indirect github.com/ipfs/go-fetcher v1.6.1 // indirect github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect @@ -203,7 +207,7 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/api v0.45.0 // indirect diff --git a/go.sum b/go.sum index 94b662e4..abadb9a7 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,12 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= +github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -275,6 +279,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -315,8 +320,12 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -500,6 +509,10 @@ github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9Dr github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-badger3 v0.0.1 h1:peMK7P8Kbf+tZ3vqDdmlmuwuwqiEh+BDhEDNuikVQlM= +github.com/ipfs/go-ds-badger3 v0.0.1/go.mod h1:krADG50uq1T7cnLnMDwAA+uEw/9QTU1hAYlc9pGVMgI= +github.com/ipfs/go-ds-badger3 v0.0.2 h1:+pME0YfRnbUKhvySnakNMuCMsUUhmGfwIsH/nnHZ7QY= +github.com/ipfs/go-ds-badger3 v0.0.2/go.mod h1:6/yjF1KaOU+IpCaqMV43yoWIdxHqOAJlO9EhWLnZSkI= github.com/ipfs/go-ds-crdt v0.3.7 h1:LVOxRa6rOUPYhDN+tFrQrE4pu7dHTuDqKT57NUWjl1Y= github.com/ipfs/go-ds-crdt v0.3.7/go.mod h1:h2hPQ3njd7DztdvUCOuV33Aq1QYRFwHXJdz+Z5oo2A0= github.com/ipfs/go-ds-crdt v0.3.8 h1:znCPI1XjCj++hckxYa7YP+udWxWkRrxZIBlEU5Ao6a0= @@ -674,6 +687,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1689,6 +1703,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/logging.go b/logging.go index 25a30a93..6842653c 100644 --- a/logging.go +++ b/logging.go @@ -43,6 +43,7 @@ var LoggingFacilitiesExtra = map[string]string{ "libp2p-raft": "FATAL", "raftlib": "ERROR", "badger": "INFO", + "badger3": "INFO", } // SetFacilityLogLevel sets the log level for a given module