From c10c37f25baad2c5157b16480272b95b21074dfa Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 25 Nov 2022 12:56:01 +0100 Subject: [PATCH 1/4] Adjust CRDT consensus timeout to be infinite (1 year) This allows for crdt migrations to finish correctly. --- cmd/ipfs-cluster-service/daemon.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/ipfs-cluster-service/daemon.go b/cmd/ipfs-cluster-service/daemon.go index 50a8bf22..e05b19cc 100644 --- a/cmd/ipfs-cluster-service/daemon.go +++ b/cmd/ipfs-cluster-service/daemon.go @@ -199,8 +199,6 @@ func createCluster( alloc, err := balanced.New(cfgs.BalancedAlloc) checkErr("creating allocator", err) - ipfscluster.ReadyTimeout = cfgs.Raft.WaitForLeaderTimeout + 5*time.Second - cons, err := setupConsensus( cfgHelper, host, @@ -290,6 +288,7 @@ func setupConsensus( if err != nil { return nil, errors.Wrap(err, "creating Raft component") } + ipfscluster.ReadyTimeout = cfgs.Raft.WaitForLeaderTimeout + 5*time.Second return rft, nil case cfgs.Crdt.ConfigKey(): convrdt, err := crdt.New( @@ -302,6 +301,9 @@ func setupConsensus( if err != nil { return nil, errors.Wrap(err, "creating CRDT component") } + // go-ds-crdt migrations are the main cause that may need + // additional time for this consensus layer to be ready. + ipfscluster.ReadyTimeout = 356 * 24 * time.Hour return convrdt, nil default: return nil, errors.New("unknown consensus component") From 970bdc8cb0f20035edcb812e36f54d0dcf80388a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 29 Nov 2022 17:09:19 +0100 Subject: [PATCH 2/4] Add Badger v3 support This commit adds badger v3 support as datastore backend for IPFS Cluster. Badgerv3 has a number of improvements over v1, particularly around compaction. It is significantly more modern and should behave better in general (some testing is pending). --- cmd/ipfs-cluster-service/main.go | 6 +- cmdutils/configs.go | 11 +- datastore/badger/badger.go | 3 + datastore/badger/config.go | 1 + datastore/badger3/badger.go | 40 ++++ datastore/badger3/config.go | 360 +++++++++++++++++++++++++++++++ datastore/badger3/config_test.go | 90 ++++++++ go.mod | 10 +- go.sum | 16 ++ logging.go | 1 + 10 files changed, 531 insertions(+), 7 deletions(-) create mode 100644 datastore/badger3/badger.go create mode 100644 datastore/badger3/config.go create mode 100644 datastore/badger3/config_test.go diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 1a4bb1d8..d229f465 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -269,12 +269,12 @@ the peer IDs in the given multiaddresses. Flags: []cli.Flag{ cli.StringFlag{ Name: "consensus", - Usage: "select consensus component: 'crdt' or 'raft'", + Usage: "select consensus: 'crdt' or 'raft'", Value: defaultConsensus, }, cli.StringFlag{ Name: "datastore", - Usage: "select datastore component: 'badger' or 'leveldb'", + Usage: "select datastore: 'badger', 'badger3' or 'leveldb'", Value: defaultDatastore, }, cli.BoolFlag{ @@ -304,7 +304,7 @@ the peer IDs in the given multiaddresses. datastore := c.String("datastore") switch datastore { - case "leveldb", "badger": + case "leveldb", "badger", "badger3": default: checkErr("choosing datastore", errors.New("flag value must be set to 'leveldb' or 'badger'")) } diff --git a/cmdutils/configs.go b/cmdutils/configs.go index 46c02dfa..f658bdc0 100644 --- a/cmdutils/configs.go +++ b/cmdutils/configs.go @@ -16,6 +16,7 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/informer/numpin" @@ -46,6 +47,7 @@ type Configs struct { Metrics *observations.MetricsConfig Tracing *observations.TracingConfig Badger *badger.Config + Badger3 *badger3.Config LevelDB *leveldb.Config } @@ -195,10 +197,11 @@ func (ch *ConfigHelper) GetDatastore() string { } badgerLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Badger.ConfigKey()) + badger3Loaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Badger3.ConfigKey()) levelDBLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.LevelDB.ConfigKey()) nLoaded := 0 - for _, v := range []bool{badgerLoaded, levelDBLoaded} { + for _, v := range []bool{badgerLoaded, badger3Loaded, levelDBLoaded} { if v { nLoaded++ } @@ -209,6 +212,8 @@ func (ch *ConfigHelper) GetDatastore() string { switch { case badgerLoaded: return ch.configs.Badger.ConfigKey() + case badger3Loaded: + return ch.configs.Badger3.ConfigKey() case levelDBLoaded: return ch.configs.LevelDB.ConfigKey() default: @@ -237,6 +242,7 @@ func (ch *ConfigHelper) init() { Metrics: &observations.MetricsConfig{}, Tracing: &observations.TracingConfig{}, Badger: &badger.Config{}, + Badger3: &badger3.Config{}, LevelDB: &leveldb.Config{}, } man.RegisterComponent(config.Cluster, cfgs.Cluster) @@ -272,12 +278,15 @@ func (ch *ConfigHelper) init() { switch ch.datastore { case cfgs.Badger.ConfigKey(): man.RegisterComponent(config.Datastore, cfgs.Badger) + case cfgs.Badger3.ConfigKey(): + man.RegisterComponent(config.Datastore, cfgs.Badger3) case cfgs.LevelDB.ConfigKey(): man.RegisterComponent(config.Datastore, cfgs.LevelDB) default: man.RegisterComponent(config.Datastore, cfgs.LevelDB) man.RegisterComponent(config.Datastore, cfgs.Badger) + man.RegisterComponent(config.Datastore, cfgs.Badger3) } } diff --git a/datastore/badger/badger.go b/datastore/badger/badger.go index 2a2fabd3..26e0e90d 100644 --- a/datastore/badger/badger.go +++ b/datastore/badger/badger.go @@ -7,9 +7,12 @@ import ( ds "github.com/ipfs/go-datastore" badgerds "github.com/ipfs/go-ds-badger" + logging "github.com/ipfs/go-log/v2" "github.com/pkg/errors" ) +var logger = logging.Logger("badger") + // New returns a BadgerDB datastore configured with the given // configuration. func New(cfg *Config) (ds.Datastore, error) { diff --git a/datastore/badger/config.go b/datastore/badger/config.go index 1b57531b..2f6ede0d 100644 --- a/datastore/badger/config.go +++ b/datastore/badger/config.go @@ -169,6 +169,7 @@ func (cfg *Config) Default() error { cfg.GCInterval = DefaultGCInterval cfg.GCSleep = DefaultGCSleep cfg.BadgerOptions = DefaultBadgerOptions + cfg.BadgerOptions.Logger = logger return nil } diff --git a/datastore/badger3/badger.go b/datastore/badger3/badger.go new file mode 100644 index 00000000..04f8cdf3 --- /dev/null +++ b/datastore/badger3/badger.go @@ -0,0 +1,40 @@ +// Package badger3 provides a configurable BadgerDB v3 go-datastore for use with +// IPFS Cluster. +package badger3 + +import ( + "os" + + ds "github.com/ipfs/go-datastore" + badgerds "github.com/ipfs/go-ds-badger3" + logging "github.com/ipfs/go-log/v2" + "github.com/pkg/errors" +) + +var logger = logging.Logger("badger3") + +// New returns a BadgerDB datastore configured with the given +// configuration. +func New(cfg *Config) (ds.Datastore, error) { + folder := cfg.GetFolder() + err := os.MkdirAll(folder, 0700) + if err != nil { + return nil, errors.Wrap(err, "creating badger folder") + } + opts := badgerds.Options{ + GcDiscardRatio: cfg.GCDiscardRatio, + GcInterval: cfg.GCInterval, + GcSleep: cfg.GCSleep, + Options: cfg.BadgerOptions, + } + return badgerds.NewDatastore(folder, &opts) +} + +// Cleanup deletes the badger datastore. +func Cleanup(cfg *Config) error { + folder := cfg.GetFolder() + if _, err := os.Stat(folder); os.IsNotExist(err) { + return nil + } + return os.RemoveAll(cfg.GetFolder()) +} diff --git a/datastore/badger3/config.go b/datastore/badger3/config.go new file mode 100644 index 00000000..adf28d4e --- /dev/null +++ b/datastore/badger3/config.go @@ -0,0 +1,360 @@ +package badger3 + +import ( + "encoding/json" + "errors" + "path/filepath" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v3/options" + "github.com/imdario/mergo" + "github.com/kelseyhightower/envconfig" + + "github.com/ipfs-cluster/ipfs-cluster/config" +) + +const configKey = "badger3" +const envConfigKey = "cluster_badger3" + +// Default values for badger Config +const ( + DefaultSubFolder = "badger3" +) + +var ( + // DefaultBadgerOptions has to be a var because badger.DefaultOptions + // is. Values are customized during Init(). + DefaultBadgerOptions badger.Options + + // DefaultGCDiscardRatio for GC operations. See Badger docs. + DefaultGCDiscardRatio float64 = 0.2 + // DefaultGCInterval specifies interval between GC cycles. + DefaultGCInterval time.Duration = 15 * time.Minute + // DefaultGCSleep specifies sleep time between GC rounds. + DefaultGCSleep time.Duration = 10 * time.Second +) + +func init() { + DefaultBadgerOptions = badger.DefaultOptions("") + // Better to slow down starts than shutdowns. + DefaultBadgerOptions.CompactL0OnClose = false + // Defaults to 1MB! For us that means everything goes into the LSM + // tree and the LSM tree is supposed to be loaded into memory in full. + // We only put very small things on the LSM tree by default (i.e. a + // single CID). + DefaultBadgerOptions.ValueThreshold = 100 + // Disable Block Cache: the cluster read-pattern at scale requires + // looping regularly all keys. The CRDT read-patterm avoids reading + // something twice. In general, it probably does not add much, and it + // is recommended to be disabled when not using compression. + DefaultBadgerOptions.BlockCacheSize = 0 + // Let's disable compression for values, better perf when reading and + // usually the ratio between data stored by badger and the cluster + // should be small. Users can always enable. + DefaultBadgerOptions.Compression = options.None + // There is a write lock in go-ds-crdt that writes batches one by one. + // Also NewWriteBatch says that there can never be transaction + // conflicts when doing batches. And IPFS will only write a block + // once, or do it with the same values. In general, we probably don't + // care about conflicts much (rows updated while a commit transaction + // was open). Increases perf too. + DefaultBadgerOptions.DetectConflicts = false + // TODO: Increase memtable size. This will use some more memory, but any + // normal system should be able to deal with using 256MiB for the + // memtable. Badger puts a lot of things in memory anyways, + // i.e. IndexCacheSize is set to 0. Note NumMemTables is 5. + // DefaultBadgerOptions.MemTableSize = 268435456 // 256MiB + +} + +// Config is used to initialize a BadgerDB datastore. It implements the +// ComponentConfig interface. +type Config struct { + config.Saver + + // The folder for this datastore. Non-absolute paths are relative to + // the base configuration folder. + Folder string + + // For GC operation. See Badger documentation. + GCDiscardRatio float64 + + // Interval between GC cycles. Each GC cycle runs one or more + // rounds separated by GCSleep. + GCInterval time.Duration + + // Time between rounds in a GC cycle + GCSleep time.Duration + + BadgerOptions badger.Options +} + +// badgerOptions is a copy of badger.Options so it can be marshaled by us. +type badgerOptions struct { + Dir string `json:"dir"` + ValueDir string `json:"value_dir"` + SyncWrites bool `json:"sync_writes"` + NumVersionsToKeep int `json:"num_versions_to_keep"` + ReadOnly bool `json:"read_only"` + // Logger + Compression options.CompressionType `json:"compression"` + InMemory bool `json:"in_memory"` + MetricsEnabled bool `json:"metrics_enabled"` + NumGoroutines int `json:"num_goroutines"` + + MemTableSize int64 `json:"mem_table_size"` + BaseTableSize int64 `json:"base_table_size"` + BaseLevelSize int64 `json:"base_level_size"` + LevelSizeMultiplier int `json:"level_size_multiplier"` + TableSizeMultiplier int `json:"table_size_multiplier"` + MaxLevels int `json:"max_levels"` + + VLogPercentile float64 `json:"v_log_percentile"` + ValueThreshold int64 `json:"value_threshold"` + NumMemtables int `json:"num_memtables"` + BlockSize int `json:"block_size"` + BloomFalsePositive float64 `json:"bloom_false_positive"` + BlockCacheSize int64 `json:"block_cache_size"` + IndexCacheSize int64 `json:"index_cache_size"` + + NumLevelZeroTables int `json:"num_level_zero_tables"` + NumLevelZeroTablesStall int `json:"num_level_zero_tables_stall"` + + ValueLogFileSize int64 `json:"value_log_file_size"` + ValueLogMaxEntries uint32 `json:"value_log_max_entries"` + + NumCompactors int `json:"num_compactors"` + CompactL0OnClose bool `json:"compact_l_0_on_close"` + LmaxCompaction bool `json:"lmax_compaction"` + ZSTDCompressionLevel int `json:"zstd_compression_level"` + + VerifyValueChecksum bool `json:"verify_value_checksum"` + + ChecksumVerificationMode options.ChecksumVerificationMode `json:"checksum_verification_mode"` + DetectConflicts bool `json:"detect_conflicts"` + + NamespaceOffset int `json:"namespace_offset"` +} + +func (bo *badgerOptions) Unmarshal() *badger.Options { + badgerOpts := &badger.Options{} + badgerOpts.Dir = bo.Dir + badgerOpts.ValueDir = bo.ValueDir + badgerOpts.SyncWrites = bo.SyncWrites + badgerOpts.NumVersionsToKeep = bo.NumVersionsToKeep + badgerOpts.ReadOnly = bo.ReadOnly + badgerOpts.Compression = bo.Compression + badgerOpts.InMemory = bo.InMemory + badgerOpts.MetricsEnabled = bo.MetricsEnabled + badgerOpts.NumGoroutines = bo.NumGoroutines + + badgerOpts.MemTableSize = bo.MemTableSize + badgerOpts.BaseTableSize = bo.BaseTableSize + badgerOpts.BaseLevelSize = bo.BaseLevelSize + badgerOpts.LevelSizeMultiplier = bo.LevelSizeMultiplier + badgerOpts.TableSizeMultiplier = bo.TableSizeMultiplier + badgerOpts.MaxLevels = bo.MaxLevels + + badgerOpts.VLogPercentile = bo.VLogPercentile + badgerOpts.ValueThreshold = bo.ValueThreshold + badgerOpts.NumMemtables = bo.NumMemtables + badgerOpts.BlockSize = bo.BlockSize + badgerOpts.BloomFalsePositive = bo.BloomFalsePositive + badgerOpts.BlockCacheSize = bo.BlockCacheSize + badgerOpts.IndexCacheSize = bo.IndexCacheSize + + badgerOpts.NumLevelZeroTables = bo.NumLevelZeroTables + badgerOpts.NumLevelZeroTablesStall = bo.NumLevelZeroTablesStall + + badgerOpts.ValueLogFileSize = bo.ValueLogFileSize + badgerOpts.ValueLogMaxEntries = bo.ValueLogMaxEntries + + badgerOpts.NumCompactors = bo.NumCompactors + badgerOpts.CompactL0OnClose = bo.CompactL0OnClose + badgerOpts.LmaxCompaction = bo.LmaxCompaction + badgerOpts.ZSTDCompressionLevel = bo.ZSTDCompressionLevel + + badgerOpts.VerifyValueChecksum = bo.VerifyValueChecksum + + badgerOpts.ChecksumVerificationMode = bo.ChecksumVerificationMode + badgerOpts.DetectConflicts = bo.DetectConflicts + + badgerOpts.NamespaceOffset = bo.NamespaceOffset + + return badgerOpts +} + +func (bo *badgerOptions) Marshal(badgerOpts *badger.Options) { + bo.Dir = badgerOpts.Dir + bo.ValueDir = badgerOpts.ValueDir + bo.SyncWrites = badgerOpts.SyncWrites + bo.NumVersionsToKeep = badgerOpts.NumVersionsToKeep + bo.ReadOnly = badgerOpts.ReadOnly + bo.Compression = badgerOpts.Compression + bo.InMemory = badgerOpts.InMemory + bo.MetricsEnabled = badgerOpts.MetricsEnabled + bo.NumGoroutines = badgerOpts.NumGoroutines + + bo.MemTableSize = badgerOpts.MemTableSize + bo.BaseTableSize = badgerOpts.BaseTableSize + bo.BaseLevelSize = badgerOpts.BaseLevelSize + bo.LevelSizeMultiplier = badgerOpts.LevelSizeMultiplier + bo.TableSizeMultiplier = badgerOpts.TableSizeMultiplier + bo.MaxLevels = badgerOpts.MaxLevels + + bo.VLogPercentile = badgerOpts.VLogPercentile + bo.ValueThreshold = badgerOpts.ValueThreshold + bo.NumMemtables = badgerOpts.NumMemtables + bo.BlockSize = badgerOpts.BlockSize + bo.BloomFalsePositive = badgerOpts.BloomFalsePositive + bo.BlockCacheSize = badgerOpts.BlockCacheSize + bo.IndexCacheSize = badgerOpts.IndexCacheSize + + bo.NumLevelZeroTables = badgerOpts.NumLevelZeroTables + bo.NumLevelZeroTablesStall = badgerOpts.NumLevelZeroTablesStall + + bo.ValueLogFileSize = badgerOpts.ValueLogFileSize + bo.ValueLogMaxEntries = badgerOpts.ValueLogMaxEntries + + bo.NumCompactors = badgerOpts.NumCompactors + bo.CompactL0OnClose = badgerOpts.CompactL0OnClose + bo.LmaxCompaction = badgerOpts.LmaxCompaction + bo.ZSTDCompressionLevel = badgerOpts.ZSTDCompressionLevel + + bo.VerifyValueChecksum = badgerOpts.VerifyValueChecksum + + bo.ChecksumVerificationMode = badgerOpts.ChecksumVerificationMode + bo.DetectConflicts = badgerOpts.DetectConflicts + + bo.NamespaceOffset = badgerOpts.NamespaceOffset +} + +type jsonConfig struct { + Folder string `json:"folder,omitempty"` + GCDiscardRatio float64 `json:"gc_discard_ratio"` + GCInterval string `json:"gc_interval"` + GCSleep string `json:"gc_sleep"` + BadgerOptions badgerOptions `json:"badger_options,omitempty"` +} + +// ConfigKey returns a human-friendly identifier for this type of Datastore. +func (cfg *Config) ConfigKey() string { + return configKey +} + +// Default initializes this Config with sensible values. +func (cfg *Config) Default() error { + cfg.Folder = DefaultSubFolder + cfg.GCDiscardRatio = DefaultGCDiscardRatio + cfg.GCInterval = DefaultGCInterval + cfg.GCSleep = DefaultGCSleep + cfg.BadgerOptions = DefaultBadgerOptions + cfg.BadgerOptions.Logger = logger + return nil +} + +// ApplyEnvVars fills in any Config fields found as environment variables. +func (cfg *Config) ApplyEnvVars() error { + jcfg := cfg.toJSONConfig() + + err := envconfig.Process(envConfigKey, jcfg) + if err != nil { + return err + } + + return cfg.applyJSONConfig(jcfg) +} + +// Validate checks that the fields of this Config have working values, +// at least in appearance. +func (cfg *Config) Validate() error { + if cfg.Folder == "" { + return errors.New("folder is unset") + } + + if cfg.GCDiscardRatio <= 0 || cfg.GCDiscardRatio >= 1 { + return errors.New("gc_discard_ratio must be more than 0 and less than 1") + } + + return nil +} + +// LoadJSON reads the fields of this Config from a JSON byteslice as +// generated by ToJSON. +func (cfg *Config) LoadJSON(raw []byte) error { + jcfg := &jsonConfig{} + err := json.Unmarshal(raw, jcfg) + if err != nil { + return err + } + cfg.Default() + + return cfg.applyJSONConfig(jcfg) +} + +func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { + config.SetIfNotDefault(jcfg.Folder, &cfg.Folder) + + // 0 is an invalid option anyways. In that case, set default (0.2) + config.SetIfNotDefault(jcfg.GCDiscardRatio, &cfg.GCDiscardRatio) + + // If these durations are set, GC is enabled by default with default + // values. + err := config.ParseDurations("badger", + &config.DurationOpt{Duration: jcfg.GCInterval, Dst: &cfg.GCInterval, Name: "gc_interval"}, + &config.DurationOpt{Duration: jcfg.GCSleep, Dst: &cfg.GCSleep, Name: "gc_sleep"}, + ) + if err != nil { + return err + } + + badgerOpts := jcfg.BadgerOptions.Unmarshal() + + if err := mergo.Merge(&cfg.BadgerOptions, badgerOpts, mergo.WithOverride); err != nil { + return err + } + + return cfg.Validate() +} + +// ToJSON generates a JSON-formatted human-friendly representation of this +// Config. +func (cfg *Config) ToJSON() (raw []byte, err error) { + jcfg := cfg.toJSONConfig() + + raw, err = config.DefaultJSONMarshal(jcfg) + return +} + +func (cfg *Config) toJSONConfig() *jsonConfig { + jCfg := &jsonConfig{} + + if cfg.Folder != DefaultSubFolder { + jCfg.Folder = cfg.Folder + } + + jCfg.GCDiscardRatio = cfg.GCDiscardRatio + jCfg.GCInterval = cfg.GCInterval.String() + jCfg.GCSleep = cfg.GCSleep.String() + + bo := &badgerOptions{} + bo.Marshal(&cfg.BadgerOptions) + jCfg.BadgerOptions = *bo + + return jCfg +} + +// GetFolder returns the BadgerDB folder. +func (cfg *Config) GetFolder() string { + if filepath.IsAbs(cfg.Folder) { + return cfg.Folder + } + + return filepath.Join(cfg.BaseDir, cfg.Folder) +} + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/datastore/badger3/config_test.go b/datastore/badger3/config_test.go new file mode 100644 index 00000000..a2110d71 --- /dev/null +++ b/datastore/badger3/config_test.go @@ -0,0 +1,90 @@ +package badger3 + +import ( + "testing" + "time" + + "github.com/dgraph-io/badger/v3" + "github.com/dgraph-io/badger/v3/options" +) + +var cfgJSON = []byte(` +{ + "folder": "test", + "gc_discard_ratio": 0.1, + "gc_sleep": "2m", + "badger_options": { + "max_levels": 4, + "compression": 2 + } +} +`) + +func TestLoadJSON(t *testing.T) { + cfg := &Config{} + err := cfg.LoadJSON(cfgJSON) + if err != nil { + t.Fatal(err) + } +} + +func TestToJSON(t *testing.T) { + cfg := &Config{} + cfg.LoadJSON(cfgJSON) + + if cfg.GCDiscardRatio != 0.1 { + t.Fatal("GCDiscardRatio should be 0.1") + } + + if cfg.GCInterval != DefaultGCInterval { + t.Fatal("GCInterval should default as it is unset") + } + + if cfg.GCSleep != 2*time.Minute { + t.Fatal("GCSleep should be 2m") + } + + if cfg.BadgerOptions.Compression != options.ZSTD { + t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.Compression, options.ZSTD) + } + + if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions("").ValueLogFileSize { + t.Fatalf( + "got: %d, want: %d", + cfg.BadgerOptions.ValueLogFileSize, + badger.DefaultOptions("").ValueLogFileSize, + ) + } + + if cfg.BadgerOptions.ChecksumVerificationMode != badger.DefaultOptions("").ChecksumVerificationMode { + t.Fatalf("ChecksumVerificationMode is not nil: got: %v, want: %v", cfg.BadgerOptions.ChecksumVerificationMode, badger.DefaultOptions("").ChecksumVerificationMode) + } + + if cfg.BadgerOptions.MaxLevels != 4 { + t.Fatalf("MaxLevels should be 4, got: %d", cfg.BadgerOptions.MaxLevels) + } + + newjson, err := cfg.ToJSON() + if err != nil { + t.Fatal(err) + } + + cfg = &Config{} + err = cfg.LoadJSON(newjson) + if err != nil { + t.Fatal(err) + } +} + +func TestDefault(t *testing.T) { + cfg := &Config{} + cfg.Default() + if cfg.Validate() != nil { + t.Fatal("error validating") + } + + cfg.GCDiscardRatio = 0 + if cfg.Validate() == nil { + t.Fatal("expected error validating") + } +} diff --git a/go.mod b/go.mod index 1b45fc36..11aba051 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,8 @@ require ( github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/dgraph-io/ristretto v0.0.2 // indirect + github.com/dgraph-io/badger/v3 v3.2103.4 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/fatih/color v1.13.0 // indirect @@ -100,9 +101,11 @@ require ( github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -117,6 +120,7 @@ require ( github.com/ipfs/go-bitswap v0.10.0 // indirect github.com/ipfs/go-blockservice v0.4.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect + github.com/ipfs/go-ds-badger3 v0.0.2 // indirect github.com/ipfs/go-fetcher v1.6.1 // indirect github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect @@ -203,7 +207,7 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/api v0.45.0 // indirect diff --git a/go.sum b/go.sum index 94b662e4..abadb9a7 100644 --- a/go.sum +++ b/go.sum @@ -187,8 +187,12 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= +github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -275,6 +279,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -315,8 +320,12 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -500,6 +509,10 @@ github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9Dr github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-badger3 v0.0.1 h1:peMK7P8Kbf+tZ3vqDdmlmuwuwqiEh+BDhEDNuikVQlM= +github.com/ipfs/go-ds-badger3 v0.0.1/go.mod h1:krADG50uq1T7cnLnMDwAA+uEw/9QTU1hAYlc9pGVMgI= +github.com/ipfs/go-ds-badger3 v0.0.2 h1:+pME0YfRnbUKhvySnakNMuCMsUUhmGfwIsH/nnHZ7QY= +github.com/ipfs/go-ds-badger3 v0.0.2/go.mod h1:6/yjF1KaOU+IpCaqMV43yoWIdxHqOAJlO9EhWLnZSkI= github.com/ipfs/go-ds-crdt v0.3.7 h1:LVOxRa6rOUPYhDN+tFrQrE4pu7dHTuDqKT57NUWjl1Y= github.com/ipfs/go-ds-crdt v0.3.7/go.mod h1:h2hPQ3njd7DztdvUCOuV33Aq1QYRFwHXJdz+Z5oo2A0= github.com/ipfs/go-ds-crdt v0.3.8 h1:znCPI1XjCj++hckxYa7YP+udWxWkRrxZIBlEU5Ao6a0= @@ -674,6 +687,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1689,6 +1703,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/logging.go b/logging.go index 25a30a93..6842653c 100644 --- a/logging.go +++ b/logging.go @@ -43,6 +43,7 @@ var LoggingFacilitiesExtra = map[string]string{ "libp2p-raft": "FATAL", "raftlib": "ERROR", "badger": "INFO", + "badger3": "INFO", } // SetFacilityLogLevel sets the log level for a given module From f496e522cde9f370d3b70f39f430b3e24fb9d613 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Dec 2022 16:58:32 +0100 Subject: [PATCH 3/4] Add Pebble support as datastore backend Pebble is a modern datastore based on RocksDB. This commit adds support to use it as datastore backend for IPFS-Cluster. --- cmd/ipfs-cluster-service/main.go | 6 +- cmdutils/configs.go | 14 +- cmdutils/state.go | 6 + datastore/pebble/config.go | 251 +++++++++++++++++++++++++++++++ datastore/pebble/config_test.go | 84 +++++++++++ datastore/pebble/pebble.go | 34 +++++ go.mod | 21 ++- go.sum | 120 +++++++++++++-- 8 files changed, 514 insertions(+), 22 deletions(-) create mode 100644 datastore/pebble/config.go create mode 100644 datastore/pebble/config_test.go create mode 100644 datastore/pebble/pebble.go diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index d229f465..16fe1eb3 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -274,7 +274,7 @@ the peer IDs in the given multiaddresses. }, cli.StringFlag{ Name: "datastore", - Usage: "select datastore: 'badger', 'badger3' or 'leveldb'", + Usage: "select datastore: 'badger', 'badger3', 'leveldb', 'pebble'", Value: defaultDatastore, }, cli.BoolFlag{ @@ -304,9 +304,9 @@ the peer IDs in the given multiaddresses. datastore := c.String("datastore") switch datastore { - case "leveldb", "badger", "badger3": + case "leveldb", "badger", "badger3", "pebble": default: - checkErr("choosing datastore", errors.New("flag value must be set to 'leveldb' or 'badger'")) + checkErr("choosing datastore", errors.New("flag value must be set to 'leveldb', 'badger', 'badger3' or 'pebble'")) } cfgHelper := cmdutils.NewConfigHelper(configPath, identityPath, consensus, datastore) diff --git a/cmdutils/configs.go b/cmdutils/configs.go index f658bdc0..27231cf3 100644 --- a/cmdutils/configs.go +++ b/cmdutils/configs.go @@ -18,6 +18,7 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" + "github.com/ipfs-cluster/ipfs-cluster/datastore/pebble" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/informer/numpin" "github.com/ipfs-cluster/ipfs-cluster/informer/pinqueue" @@ -49,6 +50,7 @@ type Configs struct { Badger *badger.Config Badger3 *badger3.Config LevelDB *leveldb.Config + Pebble *pebble.Config } // ConfigHelper helps managing the configuration and identity files with the @@ -199,9 +201,10 @@ func (ch *ConfigHelper) GetDatastore() string { badgerLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Badger.ConfigKey()) badger3Loaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Badger3.ConfigKey()) levelDBLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.LevelDB.ConfigKey()) + pebbleLoaded := ch.manager.IsLoadedFromJSON(config.Datastore, ch.configs.Pebble.ConfigKey()) nLoaded := 0 - for _, v := range []bool{badgerLoaded, badger3Loaded, levelDBLoaded} { + for _, v := range []bool{badgerLoaded, badger3Loaded, levelDBLoaded, pebbleLoaded} { if v { nLoaded++ } @@ -216,6 +219,8 @@ func (ch *ConfigHelper) GetDatastore() string { return ch.configs.Badger3.ConfigKey() case levelDBLoaded: return ch.configs.LevelDB.ConfigKey() + case pebbleLoaded: + return ch.configs.Pebble.ConfigKey() default: return "" } @@ -244,6 +249,7 @@ func (ch *ConfigHelper) init() { Badger: &badger.Config{}, Badger3: &badger3.Config{}, LevelDB: &leveldb.Config{}, + Pebble: &pebble.Config{}, } man.RegisterComponent(config.Cluster, cfgs.Cluster) man.RegisterComponent(config.API, cfgs.Restapi) @@ -282,11 +288,15 @@ func (ch *ConfigHelper) init() { man.RegisterComponent(config.Datastore, cfgs.Badger3) case cfgs.LevelDB.ConfigKey(): man.RegisterComponent(config.Datastore, cfgs.LevelDB) + case cfgs.Pebble.ConfigKey(): + man.RegisterComponent(config.Datastore, cfgs.Pebble) default: - man.RegisterComponent(config.Datastore, cfgs.LevelDB) man.RegisterComponent(config.Datastore, cfgs.Badger) man.RegisterComponent(config.Datastore, cfgs.Badger3) + man.RegisterComponent(config.Datastore, cfgs.LevelDB) + man.RegisterComponent(config.Datastore, cfgs.Pebble) + } } diff --git a/cmdutils/state.go b/cmdutils/state.go index fdfa5ec6..57cab659 100644 --- a/cmdutils/state.go +++ b/cmdutils/state.go @@ -13,8 +13,10 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/inmem" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" + "github.com/ipfs-cluster/ipfs-cluster/datastore/pebble" "github.com/ipfs-cluster/ipfs-cluster/pstoremgr" "github.com/ipfs-cluster/ipfs-cluster/state" @@ -126,8 +128,12 @@ func (crdtsm *crdtStateManager) GetStore() (ds.Datastore, error) { switch crdtsm.datastore { case crdtsm.cfgs.Badger.ConfigKey(): return badger.New(crdtsm.cfgs.Badger) + case crdtsm.cfgs.Badger3.ConfigKey(): + return badger3.New(crdtsm.cfgs.Badger3) case crdtsm.cfgs.LevelDB.ConfigKey(): return leveldb.New(crdtsm.cfgs.LevelDB) + case crdtsm.cfgs.Pebble.ConfigKey(): + return pebble.New(crdtsm.cfgs.Pebble) default: return nil, errors.New("unknown datastore") } diff --git a/datastore/pebble/config.go b/datastore/pebble/config.go new file mode 100644 index 00000000..4903c29a --- /dev/null +++ b/datastore/pebble/config.go @@ -0,0 +1,251 @@ +package pebble + +import ( + "encoding/json" + "errors" + "path/filepath" + "time" + + "github.com/cockroachdb/pebble" + "github.com/cockroachdb/pebble/bloom" + "github.com/imdario/mergo" + "github.com/kelseyhightower/envconfig" + + "github.com/ipfs-cluster/ipfs-cluster/config" +) + +const configKey = "pebble" +const envConfigKey = "cluster_pebble" + +// Default values for Pebble Config +const ( + DefaultSubFolder = "pebble" +) + +var ( + // DefaultPebbleOptions for convenience. + DefaultPebbleOptions pebble.Options +) + +func init() { + DefaultPebbleOptions = *DefaultPebbleOptions.EnsureDefaults() + DefaultPebbleOptions.Levels[0].Compression = pebble.NoCompression + DefaultPebbleOptions.Levels[0].FilterPolicy = bloom.FilterPolicy(10) +} + +// Config is used to initialize a Pebble datastore. It implements the +// ComponentConfig interface. +type Config struct { + config.Saver + + // The folder for this datastore. Non-absolute paths are relative to + // the base configuration folder. + Folder string + + PebbleOptions pebble.Options +} + +// pebbleOptions is a subset of pebble.Options so it can be marshaled by us in +// the cluster configuration. +type pebbleOptions struct { + BytesPerSync int `json:"bytes_per_sync"` + DisableWAL bool `json:"disable_wal"` + FlushDelayDeleteRange time.Duration `json:"flush_delay_delete_range"` + FlushDelayRangeKey time.Duration `json:"flush_delay_range_key"` + FlushSplitBytes int64 `json:"flush_split_bytes"` + FormatMajorVersion pebble.FormatMajorVersion `json:"format_major_version"` + L0CompactionFileThreshold int `json:"l0_compaction_file_threshold"` + L0CompactionThreshold int `json:"l0_compaction_threshold"` + L0StopWritesThreshold int `json:"l0_stop_writes_threshold"` + LBaseMaxBytes int64 `json:"l_base_max_bytes"` + Levels []levelOptions `json:"levels"` + MaxOpenFiles int `json:"max_open_files"` + MemTableSize int `json:"mem_table_size"` + MemTableStopWritesThreshold int `json:"mem_table_stop_writes_threshold"` + ReadOnly bool `json:"read_only"` + WALBytesPerSync int `json:"wal_bytes_per_sync"` +} + +func (po *pebbleOptions) Unmarshal() *pebble.Options { + pebbleOpts := &pebble.Options{} + pebbleOpts.BytesPerSync = po.BytesPerSync + pebbleOpts.DisableWAL = po.DisableWAL + pebbleOpts.FlushDelayDeleteRange = po.FlushDelayDeleteRange + pebbleOpts.FlushSplitBytes = po.FlushSplitBytes + pebbleOpts.FormatMajorVersion = po.FormatMajorVersion + pebbleOpts.L0CompactionFileThreshold = po.L0CompactionFileThreshold + pebbleOpts.L0CompactionThreshold = po.L0CompactionThreshold + pebbleOpts.L0StopWritesThreshold = po.L0StopWritesThreshold + pebbleOpts.LBaseMaxBytes = po.LBaseMaxBytes + pebbleOpts.Levels = make([]pebble.LevelOptions, len(po.Levels)) + for i := range po.Levels { + pebbleOpts.Levels[i] = *po.Levels[i].Unmarshal() + } + pebbleOpts.MaxOpenFiles = po.MaxOpenFiles + pebbleOpts.MemTableSize = po.MemTableSize + pebbleOpts.MemTableStopWritesThreshold = po.MemTableStopWritesThreshold + pebbleOpts.ReadOnly = po.ReadOnly + pebbleOpts.WALBytesPerSync = po.WALBytesPerSync + return pebbleOpts +} + +func (po *pebbleOptions) Marshal(pebbleOpts *pebble.Options) { + po.BytesPerSync = pebbleOpts.BytesPerSync + po.DisableWAL = pebbleOpts.DisableWAL + po.FlushDelayDeleteRange = pebbleOpts.FlushDelayDeleteRange + po.FlushSplitBytes = pebbleOpts.FlushSplitBytes + po.FormatMajorVersion = pebbleOpts.FormatMajorVersion + po.L0CompactionFileThreshold = pebbleOpts.L0CompactionFileThreshold + po.L0CompactionThreshold = pebbleOpts.L0CompactionThreshold + po.L0StopWritesThreshold = pebbleOpts.L0StopWritesThreshold + po.LBaseMaxBytes = pebbleOpts.LBaseMaxBytes + po.Levels = make([]levelOptions, len(pebbleOpts.Levels)) + for i := range pebbleOpts.Levels { + po.Levels[i].Marshal(&pebbleOpts.Levels[i]) + } + po.MaxOpenFiles = pebbleOpts.MaxOpenFiles + po.MemTableSize = pebbleOpts.MemTableSize + po.MemTableStopWritesThreshold = pebbleOpts.MemTableStopWritesThreshold + po.ReadOnly = pebbleOpts.ReadOnly + po.WALBytesPerSync = pebbleOpts.WALBytesPerSync +} + +// levelOptions carries options for pebble's per-level parameters. +type levelOptions struct { + BlockRestartInterval int `json:"block_restart_interval"` + BlockSize int `json:"block_size"` + BlockSizeThreshold int `json:"block_size_threshold"` + Compression pebble.Compression `json:"compression"` + FilterType pebble.FilterType `json:"filter_type"` + IndexBlockSize int `json:"index_block_size"` + TargetFileSize int64 `json:"target_file_size"` +} + +func (lo *levelOptions) Unmarshal() *pebble.LevelOptions { + levelOpts := &pebble.LevelOptions{} + levelOpts.BlockRestartInterval = lo.BlockRestartInterval + levelOpts.BlockSize = lo.BlockSize + levelOpts.BlockSizeThreshold = lo.BlockSizeThreshold + levelOpts.Compression = lo.Compression + levelOpts.FilterType = lo.FilterType + levelOpts.IndexBlockSize = lo.IndexBlockSize + levelOpts.TargetFileSize = lo.TargetFileSize + return levelOpts +} + +func (lo *levelOptions) Marshal(levelOpts *pebble.LevelOptions) { + lo.BlockRestartInterval = levelOpts.BlockRestartInterval + lo.BlockSize = levelOpts.BlockSize + lo.BlockSizeThreshold = levelOpts.BlockSizeThreshold + lo.Compression = levelOpts.Compression + lo.FilterType = levelOpts.FilterType + lo.IndexBlockSize = levelOpts.IndexBlockSize + lo.TargetFileSize = levelOpts.TargetFileSize +} + +type jsonConfig struct { + Folder string `json:"folder,omitempty"` + PebbleOptions pebbleOptions `json:"pebble_options,omitempty"` +} + +// ConfigKey returns a human-friendly identifier for this type of Datastore. +func (cfg *Config) ConfigKey() string { + return configKey +} + +// Default initializes this Config with sensible values. +func (cfg *Config) Default() error { + cfg.Folder = DefaultSubFolder + cfg.PebbleOptions = DefaultPebbleOptions + cfg.PebbleOptions.Logger = logger + + return nil +} + +// ApplyEnvVars fills in any Config fields found as environment variables. +func (cfg *Config) ApplyEnvVars() error { + jcfg := cfg.toJSONConfig() + + err := envconfig.Process(envConfigKey, jcfg) + if err != nil { + return err + } + + return cfg.applyJSONConfig(jcfg) +} + +// Validate checks that the fields of this Config have working values, +// at least in appearance. +func (cfg *Config) Validate() error { + if cfg.Folder == "" { + return errors.New("folder is unset") + } + + if err := cfg.PebbleOptions.Validate(); err != nil { + return err + } + + return nil +} + +// LoadJSON reads the fields of this Config from a JSON byteslice as +// generated by ToJSON. +func (cfg *Config) LoadJSON(raw []byte) error { + jcfg := &jsonConfig{} + err := json.Unmarshal(raw, jcfg) + if err != nil { + return err + } + cfg.Default() + + return cfg.applyJSONConfig(jcfg) +} + +func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { + config.SetIfNotDefault(jcfg.Folder, &cfg.Folder) + + pebbleOpts := jcfg.PebbleOptions.Unmarshal() + + if err := mergo.Merge(&cfg.PebbleOptions, pebbleOpts, mergo.WithOverride); err != nil { + return err + } + + return cfg.Validate() +} + +// ToJSON generates a JSON-formatted human-friendly representation of this +// Config. +func (cfg *Config) ToJSON() (raw []byte, err error) { + jcfg := cfg.toJSONConfig() + + raw, err = config.DefaultJSONMarshal(jcfg) + return +} + +func (cfg *Config) toJSONConfig() *jsonConfig { + jCfg := &jsonConfig{} + + if cfg.Folder != DefaultSubFolder { + jCfg.Folder = cfg.Folder + } + + po := &pebbleOptions{} + po.Marshal(&cfg.PebbleOptions) + jCfg.PebbleOptions = *po + + return jCfg +} + +// GetFolder returns the Pebble folder. +func (cfg *Config) GetFolder() string { + if filepath.IsAbs(cfg.Folder) { + return cfg.Folder + } + + return filepath.Join(cfg.BaseDir, cfg.Folder) +} + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/datastore/pebble/config_test.go b/datastore/pebble/config_test.go new file mode 100644 index 00000000..7e6d1e5b --- /dev/null +++ b/datastore/pebble/config_test.go @@ -0,0 +1,84 @@ +package pebble + +import ( + "testing" +) + +var cfgJSON = []byte(` +{ + "folder": "test", + "pebble_options": { + "bytes_per_sync": 524288, + "disable_wal": true, + "flush_delay_delete_range": 0, + "flush_delay_range_key": 0, + "flush_split_bytes": 4194304, + "format_major_version": 1, + "l0_compaction_file_threshold": 500, + "l0_compaction_threshold": 2, + "l0_stop_writes_threshold": 12, + "l_base_max_bytes": 67108864, + "levels": [ + { + "block_restart_interval": 16, + "block_size": 4096, + "block_size_threshold": 90, + "Compression": 2, + "filter_type": 0, + "index_block_size": 8000, + "target_file_size": 2097152 + } + ], + "max_open_files": 1000, + "mem_table_size": 4194304, + "mem_table_stop_writes_threshold": 2, + "read_only": false, + "wal_bytes_per_sync": 0 + } +} +`) + +func TestLoadJSON(t *testing.T) { + cfg := &Config{} + err := cfg.LoadJSON(cfgJSON) + if err != nil { + t.Fatal(err) + } +} + +func TestToJSON(t *testing.T) { + cfg := &Config{} + cfg.LoadJSON(cfgJSON) + + if cfg.PebbleOptions.L0CompactionThreshold != 2 { + t.Fatalf("got: %d, want: %d", cfg.PebbleOptions.L0CompactionThreshold, 2) + } + + if !cfg.PebbleOptions.DisableWAL { + t.Fatal("Disable WAL should be true") + } + + newjson, err := cfg.ToJSON() + if err != nil { + t.Fatal(err) + } + + cfg = &Config{} + err = cfg.LoadJSON(newjson) + if err != nil { + t.Fatal(err) + } +} + +func TestDefault(t *testing.T) { + cfg := &Config{} + cfg.Default() + if cfg.Validate() != nil { + t.Fatal("error validating") + } + + cfg.PebbleOptions.MemTableStopWritesThreshold = 0 + if cfg.Validate() == nil { + t.Fatal("expected error validating") + } +} diff --git a/datastore/pebble/pebble.go b/datastore/pebble/pebble.go new file mode 100644 index 00000000..ca785ac7 --- /dev/null +++ b/datastore/pebble/pebble.go @@ -0,0 +1,34 @@ +// Package pebble provides a configurable Pebble database backend for use with +// IPFS Cluster. +package pebble + +import ( + "os" + + ds "github.com/ipfs/go-datastore" + pebbleds "github.com/ipfs/go-ds-pebble" + logging "github.com/ipfs/go-log/v2" + "github.com/pkg/errors" +) + +var logger = logging.Logger("pebble") + +// New returns a BadgerDB datastore configured with the given +// configuration. +func New(cfg *Config) (ds.Datastore, error) { + folder := cfg.GetFolder() + err := os.MkdirAll(folder, 0700) + if err != nil { + return nil, errors.Wrap(err, "creating pebble folder") + } + return pebbleds.NewDatastore(folder, &cfg.PebbleOptions) +} + +// Cleanup deletes the badger datastore. +func Cleanup(cfg *Config) error { + folder := cfg.GetFolder() + if _, err := os.Stat(folder); os.IsNotExist(err) { + return nil + } + return os.RemoveAll(cfg.GetFolder()) +} diff --git a/go.mod b/go.mod index 11aba051..bec6562a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ require ( contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/blang/semver v3.5.1+incompatible + github.com/cockroachdb/pebble v0.0.0-20221122204154-936e011bb911 github.com/dgraph-io/badger v1.6.2 + github.com/dgraph-io/badger/v3 v3.2103.4 github.com/dustin/go-humanize v1.0.0 github.com/golang-jwt/jwt/v4 v4.4.2 github.com/google/uuid v1.3.0 @@ -18,8 +20,10 @@ require ( github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger v0.3.0 + github.com/ipfs/go-ds-badger3 v0.0.2 github.com/ipfs/go-ds-crdt v0.3.9 github.com/ipfs/go-ds-leveldb v0.5.0 + github.com/ipfs/go-ds-pebble v0.1.0 github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-ipfs-api v0.3.0 github.com/ipfs/go-ipfs-chunker v0.0.5 @@ -69,6 +73,7 @@ require ( require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect github.com/benbjohnson/clock v1.3.0 // indirect @@ -78,6 +83,10 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cheekybits/genny v1.0.0 // indirect + github.com/cockroachdb/errors v1.8.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect + github.com/cockroachdb/redact v1.0.8 // indirect + github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -85,7 +94,6 @@ require ( github.com/cskr/pubsub v1.0.2 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/dgraph-io/badger/v3 v3.2103.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect @@ -104,7 +112,7 @@ require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -120,7 +128,6 @@ require ( github.com/ipfs/go-bitswap v0.10.0 // indirect github.com/ipfs/go-blockservice v0.4.0 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect - github.com/ipfs/go-ds-badger3 v0.0.2 // indirect github.com/ipfs/go-fetcher v1.6.1 // indirect github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect @@ -142,6 +149,8 @@ require ( github.com/klauspost/compress v1.15.1 // indirect github.com/klauspost/cpuid/v2 v2.1.0 // indirect github.com/koron/go-ssdp v0.0.3 // indirect + github.com/kr/pretty v0.3.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -165,7 +174,7 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect @@ -184,11 +193,12 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -204,6 +214,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/zap v1.22.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect + golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect diff --git a/go.sum b/go.sum index abadb9a7..e4fcb5c4 100644 --- a/go.sum +++ b/go.sum @@ -53,17 +53,25 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= +github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= +github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -85,6 +93,7 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -145,7 +154,20 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= +github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20221122204154-936e011bb911 h1:DgASk5q0ER4TW8utcptn1ZP49lLjSkwYH6MTi99Jfhc= +github.com/cockroachdb/pebble v0.0.0-20221122204154-936e011bb911/go.mod h1:qf9bLis2yy1XyNYD01wvIHPabuC1STzQsvGibYVsom4= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= @@ -168,6 +190,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -189,7 +212,6 @@ github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v3 v3.2103.4 h1:WE1B07YNTTJTtG9xjBcSW2wn0RJLyiV99h959RKZqM4= github.com/dgraph-io/badger/v3 v3.2103.4/go.mod h1:4MPiseMeDQ3FNCYwRbbcBOGJLf5jsE0PPFzRiKjtcdw= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= @@ -206,6 +228,7 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= @@ -218,11 +241,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= @@ -237,9 +264,13 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -261,14 +292,19 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -277,6 +313,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -318,10 +355,11 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= @@ -445,10 +483,12 @@ github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7 github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= @@ -509,14 +549,8 @@ github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9Dr github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= -github.com/ipfs/go-ds-badger3 v0.0.1 h1:peMK7P8Kbf+tZ3vqDdmlmuwuwqiEh+BDhEDNuikVQlM= -github.com/ipfs/go-ds-badger3 v0.0.1/go.mod h1:krADG50uq1T7cnLnMDwAA+uEw/9QTU1hAYlc9pGVMgI= github.com/ipfs/go-ds-badger3 v0.0.2 h1:+pME0YfRnbUKhvySnakNMuCMsUUhmGfwIsH/nnHZ7QY= github.com/ipfs/go-ds-badger3 v0.0.2/go.mod h1:6/yjF1KaOU+IpCaqMV43yoWIdxHqOAJlO9EhWLnZSkI= -github.com/ipfs/go-ds-crdt v0.3.7 h1:LVOxRa6rOUPYhDN+tFrQrE4pu7dHTuDqKT57NUWjl1Y= -github.com/ipfs/go-ds-crdt v0.3.7/go.mod h1:h2hPQ3njd7DztdvUCOuV33Aq1QYRFwHXJdz+Z5oo2A0= -github.com/ipfs/go-ds-crdt v0.3.8 h1:znCPI1XjCj++hckxYa7YP+udWxWkRrxZIBlEU5Ao6a0= -github.com/ipfs/go-ds-crdt v0.3.8/go.mod h1:h2hPQ3njd7DztdvUCOuV33Aq1QYRFwHXJdz+Z5oo2A0= github.com/ipfs/go-ds-crdt v0.3.9 h1:hZl67DynkBWGz2YwTQjR7d/dYNMEHFpVX2bz9Dyu6k8= github.com/ipfs/go-ds-crdt v0.3.9/go.mod h1:h2hPQ3njd7DztdvUCOuV33Aq1QYRFwHXJdz+Z5oo2A0= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= @@ -524,6 +558,8 @@ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= +github.com/ipfs/go-ds-pebble v0.1.0 h1:CCgOP1LR0OA/dbK9KIWGHGse2dqQ1jiPgznioYyu6C0= +github.com/ipfs/go-ds-pebble v0.1.0/go.mod h1:7w3s1iFpBr2RkLYXpNulbqrIerqprGUjD48pvHAOGyQ= github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryWedVuSWE= github.com/ipfs/go-fetcher v1.6.1 h1:UFuRVYX5AIllTiRhi5uK/iZkfhSpBCGX7L70nSZEmK8= github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= @@ -644,6 +680,10 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.18.0 h1:xUk7NUBSWHEXdjiOu2sLXouFJOMs0yoYzeI5RAqhYQo= github.com/ipld/go-ipld-prime v0.18.0/go.mod h1:735yXW548CKrLwVCYXzqx90p5deRJMVVxM9eJ4Qe+qE= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -675,9 +715,17 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= +github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= +github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= +github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= +github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= +github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kishansagathiya/go-dot v0.1.0 h1:XPj/333a6Qn4VPFqF+e2EiyABL7yRObJ7RTAbriKA1s= @@ -687,9 +735,12 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= @@ -705,10 +756,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lanzafame/go-libp2p-ocgorpc v0.1.1 h1:yDXjQYel7WVC/oozZoJIUzHg3DMfBGVtBr+TXtM/RMs= github.com/lanzafame/go-libp2p-ocgorpc v0.1.1/go.mod h1:Naz1HcGy8RHTQtXtr2s8xDGreZRETtpOlVJqRx4ucuo= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= @@ -1019,7 +1074,9 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= @@ -1028,10 +1085,15 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= +github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -1068,6 +1130,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -1144,7 +1207,9 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= @@ -1161,6 +1226,7 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -1199,6 +1265,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1230,8 +1298,9 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= +github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1264,6 +1333,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -1273,9 +1343,12 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1366,6 +1439,11 @@ github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.14.1 h1:0Sx+C9404t2+DPuIJ3UpZFOEFhNG3wPxMj7uZHyZKFA= github.com/urfave/cli/v2 v2.14.1/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/warpfork/go-testmark v0.10.0 h1:E86YlUMYfwIacEsQGlnTvjk1IgYkyTGjPhF0RnwTCmw= @@ -1391,10 +1469,17 @@ github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1: github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1489,6 +1574,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= +golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1533,6 +1620,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1544,6 +1632,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1640,6 +1729,7 @@ golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1701,7 +1791,6 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1728,11 +1817,13 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1829,6 +1920,7 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1877,6 +1969,7 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a h1:tzkHckzMzgPr8SC4taTC3AldLr4+oJivSoq1xf/nhsc= google.golang.org/genproto v0.0.0-20210510173355-fb37daa5cd7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1934,7 +2027,10 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= From 58cad80157e35bf9b721d9a5590e6faf7ad3a1cf Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Dec 2022 17:17:54 +0100 Subject: [PATCH 4/4] Enable tests using the new datastores (badger3, pebble) --- .github/workflows/tests.yml | 33 +++++++++++++ cluster_test.go | 64 +++++++++++++----------- config_test.go | 97 ++++++++++++++++++++++++++++++++++--- ipfscluster_test.go | 40 +++++++++++---- 4 files changed, 190 insertions(+), 44 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a32b2ac4..2c1c4d78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,6 +61,39 @@ jobs: - name: "Tests" run: go test -v -timeout 15m -failfast -datastore leveldb . + tests-badger3: + name: "Using Badger3" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO }} + + - name: "Tests" + run: go test -v -timeout 15m -failfast -datastore badger3 . + + tests-pebble: + name: "Using Pebble" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO }} + + - name: "Tests" + run: go test -v -timeout 15m -failfast -datastore pebble . + + tests-check: name: "Build, syntax and spelling checks" runs-on: ubuntu-latest diff --git a/cluster_test.go b/cluster_test.go index cc220afe..6eda67cf 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -159,7 +159,7 @@ type mockTracer struct { } func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracker) { - ident, clusterCfg, _, _, _, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, _, _, _ := testingConfigs() + ident, clusterCfg, _, _, _, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, _, _, _ := testingConfigs() ctx := context.Background() host, pubsub, dht := createHost(t, ident.PrivateKey, clusterCfg.Secret, clusterCfg.ListenAddr) @@ -169,7 +169,9 @@ func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracke clusterCfg.SetBaseDir(folder) raftCfg.DataFolder = folder badgerCfg.Folder = filepath.Join(folder, "badger") + badger3Cfg.Folder = filepath.Join(folder, "badger3") levelDBCfg.Folder = filepath.Join(folder, "leveldb") + pebbleCfg.Folder = filepath.Join(folder, "pebble") api := &mockAPI{} proxy := &mockProxy{} @@ -177,7 +179,7 @@ func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracke tracer := &mockTracer{} - store := makeStore(t, badgerCfg, levelDBCfg) + store := makeStore(t, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg) cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, false, crdtCfg) tracker := stateless.New(statelesstrackerCfg, ident.ID, clusterCfg.Peername, cons.State) @@ -229,27 +231,33 @@ func cleanState() { os.RemoveAll(testsFolder) } +func shutdownTestingCluster(ctx context.Context, t *testing.T, cl *Cluster) { + t.Helper() + err := cl.Shutdown(ctx) + if err != nil { + t.Fatal("cluster shutdown failed:", err) + } + cl.dht.Close() + cl.host.Close() + cl.datastore.Close() +} + func TestClusterShutdown(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) - err := cl.Shutdown(ctx) - if err != nil { - t.Error("cluster shutdown failed:", err) - } - cl.Shutdown(ctx) + shutdownTestingCluster(ctx, t, cl) + shutdownTestingCluster(ctx, t, cl) + cl, _, _, _ = testingCluster(t) - err = cl.Shutdown(ctx) - if err != nil { - t.Error("cluster shutdown failed:", err) - } + shutdownTestingCluster(ctx, t, cl) + cleanState() } func TestClusterStateSync(t *testing.T) { ctx := context.Background() - cleanState() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -279,7 +287,7 @@ func TestClusterID(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) id := cl.ID(ctx) if len(id.Addresses) == 0 { t.Error("expected more addresses") @@ -299,7 +307,7 @@ func TestClusterPin(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 res, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -332,7 +340,7 @@ func TestPinExpired(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{ @@ -347,7 +355,7 @@ func TestClusterPinPath(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) pin, err := cl.PinPath(ctx, test.PathIPFS2, api.PinOptions{}) if err != nil { @@ -368,7 +376,7 @@ func TestAddFile(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) sth := test.NewShardingTestHelper() defer sth.Clean(t) @@ -428,7 +436,7 @@ func TestUnpinShard(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) sth := test.NewShardingTestHelper() defer sth.Clean(t) @@ -794,7 +802,7 @@ func TestClusterPins(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -820,7 +828,7 @@ func TestClusterPinGet(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -846,7 +854,7 @@ func TestClusterUnpin(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 // Unpin should error without pin being committed to state @@ -881,7 +889,7 @@ func TestClusterUnpinPath(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) // Unpin should error without pin being committed to state _, err := cl.UnpinPath(ctx, test.PathIPFS2) @@ -911,7 +919,7 @@ func TestClusterPeers(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) out := make(chan api.ID, 10) cl.Peers(ctx, out) @@ -935,7 +943,7 @@ func TestVersion(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) if cl.Version() != version.Version.String() { t.Error("bad Version()") } @@ -945,7 +953,7 @@ func TestClusterRecoverAllLocal(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) _, err := cl.Pin(ctx, test.ErrorCid, api.PinOptions{}) if err != nil { @@ -974,7 +982,7 @@ func TestClusterRepoGC(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) gRepoGC, err := cl.RepoGC(ctx) if err != nil { @@ -998,7 +1006,7 @@ func TestClusterRepoGCLocal(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) repoGC, err := cl.RepoGCLocal(ctx) if err != nil { diff --git a/config_test.go b/config_test.go index c39871bb..81b05e41 100644 --- a/config_test.go +++ b/config_test.go @@ -8,7 +8,9 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" + "github.com/ipfs-cluster/ipfs-cluster/datastore/pebble" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/ipfsconn/ipfshttp" "github.com/ipfs-cluster/ipfs-cluster/monitor/pubsubmon" @@ -74,12 +76,89 @@ var testingBadgerCfg = []byte(`{ } }`) +var testingBadger3Cfg = []byte(` +{ + "gc_discard_ratio": 0.2, + "gc_interval": "0s", + "gc_sleep": "0s", + "badger_options": { + "dir": "", + "value_dir": "", + "sync_writes": false, + "num_versions_to_keep": 1, + "read_only": false, + "compression": 0, + "in_memory": false, + "metrics_enabled": true, + "num_goroutines": 8, + "mem_table_size": 1048576, + "base_table_size": 2097152, + "base_level_size": 10485760, + "level_size_multiplier": 10, + "table_size_multiplier": 2, + "max_levels": 7, + "v_log_percentile": 0, + "value_threshold": 100, + "num_memtables": 5, + "block_size": 4096, + "bloom_false_positive": 0.01, + "block_cache_size": 0, + "index_cache_size": 0, + "num_level_zero_tables": 5, + "num_level_zero_tables_stall": 15, + "value_log_file_size": 1073741823, + "value_log_max_entries": 1000000, + "num_compactors": 4, + "compact_l_0_on_close": false, + "lmax_compaction": false, + "zstd_compression_level": 1, + "verify_value_checksum": false, + "checksum_verification_mode": 0, + "detect_conflicts": false, + "namespace_offset": -1 + } + } +`) + var testingLevelDBCfg = []byte(`{ "folder": "leveldbFromTests", "leveldb_options": { } }`) +var testingPebbleCfg = []byte(` +{ + "pebble_options": { + "bytes_per_sync": 524288, + "disable_wal": false, + "flush_delay_delete_range": 0, + "flush_delay_range_key": 0, + "flush_split_bytes": 4194304, + "format_major_version": 1, + "l0_compaction_file_threshold": 500, + "l0_compaction_threshold": 4, + "l0_stop_writes_threshold": 12, + "l_base_max_bytes": 67108864, + "levels": [ + { + "block_restart_interval": 16, + "block_size": 4096, + "block_size_threshold": 90, + "Compression": 1, + "filter_type": 0, + "index_block_size": 8000, + "target_file_size": 2097152 + } + ], + "max_open_files": 1000, + "mem_table_size": 1048576, + "mem_table_stop_writes_threshold": 2, + "read_only": false, + "wal_bytes_per_sync": 0 + } + } +`) + var testingAPICfg = []byte(`{ "http_listen_multiaddress": "/ip4/127.0.0.1/tcp/10002", "read_timeout": "0", @@ -144,16 +223,18 @@ var testingTracerCfg = []byte(`{ "service_name": "cluster-daemon" }`) -func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *leveldb.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { - identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingEmptyConfigs() +func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *badger3.Config, *leveldb.Config, *pebble.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { + identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingEmptyConfigs() identity.LoadJSON(testingIdentity) clusterCfg.LoadJSON(testingClusterCfg) apiCfg.LoadJSON(testingAPICfg) proxyCfg.LoadJSON(testingProxyCfg) ipfsCfg.LoadJSON(testingIpfsCfg) badgerCfg.LoadJSON(testingBadgerCfg) - raftCfg.LoadJSON(testingRaftCfg) + badger3Cfg.LoadJSON(testingBadger3Cfg) levelDBCfg.LoadJSON(testingLevelDBCfg) + pebbleCfg.LoadJSON(testingPebbleCfg) + raftCfg.LoadJSON(testingRaftCfg) crdtCfg.LoadJSON(testingCrdtCfg) statelesstrkrCfg.LoadJSON(testingTrackerCfg) pubsubmonCfg.LoadJSON(testingMonCfg) @@ -161,25 +242,27 @@ func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Confi diskInfCfg.LoadJSON(testingDiskInfCfg) tracingCfg.LoadJSON(testingTracerCfg) - return identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg + return identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg } -func testingEmptyConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *leveldb.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { +func testingEmptyConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *badger3.Config, *leveldb.Config, *pebble.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { identity := &config.Identity{} clusterCfg := &Config{} apiCfg := rest.NewConfig() proxyCfg := &ipfsproxy.Config{} ipfshttpCfg := &ipfshttp.Config{} badgerCfg := &badger.Config{} - raftCfg := &raft.Config{} + badger3Cfg := &badger3.Config{} levelDBCfg := &leveldb.Config{} + pebbleCfg := &pebble.Config{} + raftCfg := &raft.Config{} crdtCfg := &crdt.Config{} statelessCfg := &stateless.Config{} pubsubmonCfg := &pubsubmon.Config{} allocBalancedCfg := &balanced.Config{} diskInfCfg := &disk.Config{} tracingCfg := &observations.TracingConfig{} - return identity, clusterCfg, apiCfg, proxyCfg, ipfshttpCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelessCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg + return identity, clusterCfg, apiCfg, proxyCfg, ipfshttpCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelessCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg } // func TestConfigDefault(t *testing.T) { diff --git a/ipfscluster_test.go b/ipfscluster_test.go index 11e2cc22..f4f02411 100644 --- a/ipfscluster_test.go +++ b/ipfscluster_test.go @@ -21,8 +21,10 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/inmem" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" + "github.com/ipfs-cluster/ipfs-cluster/datastore/pebble" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/ipfsconn/ipfshttp" "github.com/ipfs-cluster/ipfs-cluster/monitor/pubsubmon" @@ -174,7 +176,7 @@ func createComponents( peername := fmt.Sprintf("peer_%d", i) - ident, clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingConfigs() + ident, clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingConfigs() ident.ID = host.ID() ident.PrivateKey = host.Peerstore().PrivKey(host.ID()) @@ -192,7 +194,9 @@ func createComponents( raftCfg.DataFolder = filepath.Join(testsFolder, host.ID().Pretty()) badgerCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger") + badger3Cfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger3") levelDBCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "leveldb") + pebbleCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "pebble") api, err := rest.NewAPI(ctx, apiCfg) if err != nil { @@ -218,7 +222,7 @@ func createComponents( t.Fatal(err) } - store := makeStore(t, badgerCfg, levelDBCfg) + store := makeStore(t, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg) cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, staging, crdtCfg) tracker := stateless.New(statelesstrackerCfg, ident.ID, clusterCfg.Peername, cons.State) @@ -239,21 +243,39 @@ func createComponents( return clusterCfg, store, cons, []API{api, ipfsProxy}, ipfs, tracker, mon, alloc, inf, tracer, mock } -func makeStore(t *testing.T, badgerCfg *badger.Config, levelDBCfg *leveldb.Config) ds.Datastore { +func makeStore(t *testing.T, badgerCfg *badger.Config, badger3Cfg *badger3.Config, levelDBCfg *leveldb.Config, pebbleCfg *pebble.Config) ds.Datastore { switch consensus { case "crdt": - if datastore == "badger" { + switch datastore { + case "badger": dstr, err := badger.New(badgerCfg) if err != nil { t.Fatal(err) } return dstr + case "badger3": + dstr, err := badger3.New(badger3Cfg) + if err != nil { + t.Fatal(err) + } + return dstr + case "leveldb": + dstr, err := leveldb.New(levelDBCfg) + if err != nil { + t.Fatal(err) + } + return dstr + case "pebble": + dstr, err := pebble.New(pebbleCfg) + if err != nil { + t.Fatal(err) + } + return dstr + default: + t.Fatal("bad datastore") + return nil } - dstr, err := leveldb.New(levelDBCfg) - if err != nil { - t.Fatal(err) - } - return dstr + default: return inmem.New() }