diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a32b2ac4..2c1c4d78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,6 +61,39 @@ jobs: - name: "Tests" run: go test -v -timeout 15m -failfast -datastore leveldb . + tests-badger3: + name: "Using Badger3" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO }} + + - name: "Tests" + run: go test -v -timeout 15m -failfast -datastore badger3 . + + tests-pebble: + name: "Using Pebble" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO }} + + - name: "Tests" + run: go test -v -timeout 15m -failfast -datastore pebble . + + tests-check: name: "Build, syntax and spelling checks" runs-on: ubuntu-latest diff --git a/cluster_test.go b/cluster_test.go index cc220afe..6eda67cf 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -159,7 +159,7 @@ type mockTracer struct { } func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracker) { - ident, clusterCfg, _, _, _, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, _, _, _ := testingConfigs() + ident, clusterCfg, _, _, _, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, _, _, _ := testingConfigs() ctx := context.Background() host, pubsub, dht := createHost(t, ident.PrivateKey, clusterCfg.Secret, clusterCfg.ListenAddr) @@ -169,7 +169,9 @@ func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracke clusterCfg.SetBaseDir(folder) raftCfg.DataFolder = folder badgerCfg.Folder = filepath.Join(folder, "badger") + badger3Cfg.Folder = filepath.Join(folder, "badger3") levelDBCfg.Folder = filepath.Join(folder, "leveldb") + pebbleCfg.Folder = filepath.Join(folder, "pebble") api := &mockAPI{} proxy := &mockProxy{} @@ -177,7 +179,7 @@ func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracke tracer := &mockTracer{} - store := makeStore(t, badgerCfg, levelDBCfg) + store := makeStore(t, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg) cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, false, crdtCfg) tracker := stateless.New(statelesstrackerCfg, ident.ID, clusterCfg.Peername, cons.State) @@ -229,27 +231,33 @@ func cleanState() { os.RemoveAll(testsFolder) } +func shutdownTestingCluster(ctx context.Context, t *testing.T, cl *Cluster) { + t.Helper() + err := cl.Shutdown(ctx) + if err != nil { + t.Fatal("cluster shutdown failed:", err) + } + cl.dht.Close() + cl.host.Close() + cl.datastore.Close() +} + func TestClusterShutdown(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) - err := cl.Shutdown(ctx) - if err != nil { - t.Error("cluster shutdown failed:", err) - } - cl.Shutdown(ctx) + shutdownTestingCluster(ctx, t, cl) + shutdownTestingCluster(ctx, t, cl) + cl, _, _, _ = testingCluster(t) - err = cl.Shutdown(ctx) - if err != nil { - t.Error("cluster shutdown failed:", err) - } + shutdownTestingCluster(ctx, t, cl) + cleanState() } func TestClusterStateSync(t *testing.T) { ctx := context.Background() - cleanState() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -279,7 +287,7 @@ func TestClusterID(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) id := cl.ID(ctx) if len(id.Addresses) == 0 { t.Error("expected more addresses") @@ -299,7 +307,7 @@ func TestClusterPin(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 res, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -332,7 +340,7 @@ func TestPinExpired(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{ @@ -347,7 +355,7 @@ func TestClusterPinPath(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) pin, err := cl.PinPath(ctx, test.PathIPFS2, api.PinOptions{}) if err != nil { @@ -368,7 +376,7 @@ func TestAddFile(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) sth := test.NewShardingTestHelper() defer sth.Clean(t) @@ -428,7 +436,7 @@ func TestUnpinShard(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) sth := test.NewShardingTestHelper() defer sth.Clean(t) @@ -794,7 +802,7 @@ func TestClusterPins(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -820,7 +828,7 @@ func TestClusterPinGet(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 _, err := cl.Pin(ctx, c, api.PinOptions{}) @@ -846,7 +854,7 @@ func TestClusterUnpin(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) c := test.Cid1 // Unpin should error without pin being committed to state @@ -881,7 +889,7 @@ func TestClusterUnpinPath(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) // Unpin should error without pin being committed to state _, err := cl.UnpinPath(ctx, test.PathIPFS2) @@ -911,7 +919,7 @@ func TestClusterPeers(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) out := make(chan api.ID, 10) cl.Peers(ctx, out) @@ -935,7 +943,7 @@ func TestVersion(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) if cl.Version() != version.Version.String() { t.Error("bad Version()") } @@ -945,7 +953,7 @@ func TestClusterRecoverAllLocal(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) _, err := cl.Pin(ctx, test.ErrorCid, api.PinOptions{}) if err != nil { @@ -974,7 +982,7 @@ func TestClusterRepoGC(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) gRepoGC, err := cl.RepoGC(ctx) if err != nil { @@ -998,7 +1006,7 @@ func TestClusterRepoGCLocal(t *testing.T) { ctx := context.Background() cl, _, _, _ := testingCluster(t) defer cleanState() - defer cl.Shutdown(ctx) + defer shutdownTestingCluster(ctx, t, cl) repoGC, err := cl.RepoGCLocal(ctx) if err != nil { diff --git a/config_test.go b/config_test.go index c39871bb..81b05e41 100644 --- a/config_test.go +++ b/config_test.go @@ -8,7 +8,9 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" + "github.com/ipfs-cluster/ipfs-cluster/datastore/pebble" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/ipfsconn/ipfshttp" "github.com/ipfs-cluster/ipfs-cluster/monitor/pubsubmon" @@ -74,12 +76,89 @@ var testingBadgerCfg = []byte(`{ } }`) +var testingBadger3Cfg = []byte(` +{ + "gc_discard_ratio": 0.2, + "gc_interval": "0s", + "gc_sleep": "0s", + "badger_options": { + "dir": "", + "value_dir": "", + "sync_writes": false, + "num_versions_to_keep": 1, + "read_only": false, + "compression": 0, + "in_memory": false, + "metrics_enabled": true, + "num_goroutines": 8, + "mem_table_size": 1048576, + "base_table_size": 2097152, + "base_level_size": 10485760, + "level_size_multiplier": 10, + "table_size_multiplier": 2, + "max_levels": 7, + "v_log_percentile": 0, + "value_threshold": 100, + "num_memtables": 5, + "block_size": 4096, + "bloom_false_positive": 0.01, + "block_cache_size": 0, + "index_cache_size": 0, + "num_level_zero_tables": 5, + "num_level_zero_tables_stall": 15, + "value_log_file_size": 1073741823, + "value_log_max_entries": 1000000, + "num_compactors": 4, + "compact_l_0_on_close": false, + "lmax_compaction": false, + "zstd_compression_level": 1, + "verify_value_checksum": false, + "checksum_verification_mode": 0, + "detect_conflicts": false, + "namespace_offset": -1 + } + } +`) + var testingLevelDBCfg = []byte(`{ "folder": "leveldbFromTests", "leveldb_options": { } }`) +var testingPebbleCfg = []byte(` +{ + "pebble_options": { + "bytes_per_sync": 524288, + "disable_wal": false, + "flush_delay_delete_range": 0, + "flush_delay_range_key": 0, + "flush_split_bytes": 4194304, + "format_major_version": 1, + "l0_compaction_file_threshold": 500, + "l0_compaction_threshold": 4, + "l0_stop_writes_threshold": 12, + "l_base_max_bytes": 67108864, + "levels": [ + { + "block_restart_interval": 16, + "block_size": 4096, + "block_size_threshold": 90, + "Compression": 1, + "filter_type": 0, + "index_block_size": 8000, + "target_file_size": 2097152 + } + ], + "max_open_files": 1000, + "mem_table_size": 1048576, + "mem_table_stop_writes_threshold": 2, + "read_only": false, + "wal_bytes_per_sync": 0 + } + } +`) + var testingAPICfg = []byte(`{ "http_listen_multiaddress": "/ip4/127.0.0.1/tcp/10002", "read_timeout": "0", @@ -144,16 +223,18 @@ var testingTracerCfg = []byte(`{ "service_name": "cluster-daemon" }`) -func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *leveldb.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { - identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingEmptyConfigs() +func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *badger3.Config, *leveldb.Config, *pebble.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { + identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingEmptyConfigs() identity.LoadJSON(testingIdentity) clusterCfg.LoadJSON(testingClusterCfg) apiCfg.LoadJSON(testingAPICfg) proxyCfg.LoadJSON(testingProxyCfg) ipfsCfg.LoadJSON(testingIpfsCfg) badgerCfg.LoadJSON(testingBadgerCfg) - raftCfg.LoadJSON(testingRaftCfg) + badger3Cfg.LoadJSON(testingBadger3Cfg) levelDBCfg.LoadJSON(testingLevelDBCfg) + pebbleCfg.LoadJSON(testingPebbleCfg) + raftCfg.LoadJSON(testingRaftCfg) crdtCfg.LoadJSON(testingCrdtCfg) statelesstrkrCfg.LoadJSON(testingTrackerCfg) pubsubmonCfg.LoadJSON(testingMonCfg) @@ -161,25 +242,27 @@ func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Confi diskInfCfg.LoadJSON(testingDiskInfCfg) tracingCfg.LoadJSON(testingTracerCfg) - return identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg + return identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg } -func testingEmptyConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *leveldb.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { +func testingEmptyConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *badger3.Config, *leveldb.Config, *pebble.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) { identity := &config.Identity{} clusterCfg := &Config{} apiCfg := rest.NewConfig() proxyCfg := &ipfsproxy.Config{} ipfshttpCfg := &ipfshttp.Config{} badgerCfg := &badger.Config{} - raftCfg := &raft.Config{} + badger3Cfg := &badger3.Config{} levelDBCfg := &leveldb.Config{} + pebbleCfg := &pebble.Config{} + raftCfg := &raft.Config{} crdtCfg := &crdt.Config{} statelessCfg := &stateless.Config{} pubsubmonCfg := &pubsubmon.Config{} allocBalancedCfg := &balanced.Config{} diskInfCfg := &disk.Config{} tracingCfg := &observations.TracingConfig{} - return identity, clusterCfg, apiCfg, proxyCfg, ipfshttpCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelessCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg + return identity, clusterCfg, apiCfg, proxyCfg, ipfshttpCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelessCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg } // func TestConfigDefault(t *testing.T) { diff --git a/ipfscluster_test.go b/ipfscluster_test.go index 11e2cc22..f4f02411 100644 --- a/ipfscluster_test.go +++ b/ipfscluster_test.go @@ -21,8 +21,10 @@ import ( "github.com/ipfs-cluster/ipfs-cluster/consensus/crdt" "github.com/ipfs-cluster/ipfs-cluster/consensus/raft" "github.com/ipfs-cluster/ipfs-cluster/datastore/badger" + "github.com/ipfs-cluster/ipfs-cluster/datastore/badger3" "github.com/ipfs-cluster/ipfs-cluster/datastore/inmem" "github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb" + "github.com/ipfs-cluster/ipfs-cluster/datastore/pebble" "github.com/ipfs-cluster/ipfs-cluster/informer/disk" "github.com/ipfs-cluster/ipfs-cluster/ipfsconn/ipfshttp" "github.com/ipfs-cluster/ipfs-cluster/monitor/pubsubmon" @@ -174,7 +176,7 @@ func createComponents( peername := fmt.Sprintf("peer_%d", i) - ident, clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingConfigs() + ident, clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingConfigs() ident.ID = host.ID() ident.PrivateKey = host.Peerstore().PrivKey(host.ID()) @@ -192,7 +194,9 @@ func createComponents( raftCfg.DataFolder = filepath.Join(testsFolder, host.ID().Pretty()) badgerCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger") + badger3Cfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger3") levelDBCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "leveldb") + pebbleCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "pebble") api, err := rest.NewAPI(ctx, apiCfg) if err != nil { @@ -218,7 +222,7 @@ func createComponents( t.Fatal(err) } - store := makeStore(t, badgerCfg, levelDBCfg) + store := makeStore(t, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg) cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, staging, crdtCfg) tracker := stateless.New(statelesstrackerCfg, ident.ID, clusterCfg.Peername, cons.State) @@ -239,21 +243,39 @@ func createComponents( return clusterCfg, store, cons, []API{api, ipfsProxy}, ipfs, tracker, mon, alloc, inf, tracer, mock } -func makeStore(t *testing.T, badgerCfg *badger.Config, levelDBCfg *leveldb.Config) ds.Datastore { +func makeStore(t *testing.T, badgerCfg *badger.Config, badger3Cfg *badger3.Config, levelDBCfg *leveldb.Config, pebbleCfg *pebble.Config) ds.Datastore { switch consensus { case "crdt": - if datastore == "badger" { + switch datastore { + case "badger": dstr, err := badger.New(badgerCfg) if err != nil { t.Fatal(err) } return dstr + case "badger3": + dstr, err := badger3.New(badger3Cfg) + if err != nil { + t.Fatal(err) + } + return dstr + case "leveldb": + dstr, err := leveldb.New(levelDBCfg) + if err != nil { + t.Fatal(err) + } + return dstr + case "pebble": + dstr, err := pebble.New(pebbleCfg) + if err != nil { + t.Fatal(err) + } + return dstr + default: + t.Fatal("bad datastore") + return nil } - dstr, err := leveldb.New(levelDBCfg) - if err != nil { - t.Fatal(err) - } - return dstr + default: return inmem.New() }