Enable tests using the new datastores (badger3, pebble)

.github/workflows/tests.yml

@@ -61,6 +61,39 @@ jobs:
     - name: "Tests"
       run: go test -v -timeout 15m -failfast -datastore leveldb .
 
+  tests-badger3:
+    name: "Using Badger3"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ env.GO }}
+
+    - name: "Tests"
+      run: go test -v -timeout 15m -failfast -datastore badger3 .
+
+  tests-pebble:
+    name: "Using Pebble"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ env.GO }}
+
+    - name: "Tests"
+      run: go test -v -timeout 15m -failfast -datastore pebble .
+
+      
   tests-check:
     name: "Build, syntax and spelling checks"
     runs-on: ubuntu-latest

cluster_test.go

@@ -159,7 +159,7 @@ type mockTracer struct {
 }
 
 func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracker) {
-	ident, clusterCfg, _, _, _, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, _, _, _ := testingConfigs()
+	ident, clusterCfg, _, _, _, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, _, _, _ := testingConfigs()
 	ctx := context.Background()
 
 	host, pubsub, dht := createHost(t, ident.PrivateKey, clusterCfg.Secret, clusterCfg.ListenAddr)
@@ -169,7 +169,9 @@ func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracke
 	clusterCfg.SetBaseDir(folder)
 	raftCfg.DataFolder = folder
 	badgerCfg.Folder = filepath.Join(folder, "badger")
+	badger3Cfg.Folder = filepath.Join(folder, "badger3")
 	levelDBCfg.Folder = filepath.Join(folder, "leveldb")
+	pebbleCfg.Folder = filepath.Join(folder, "pebble")
 
 	api := &mockAPI{}
 	proxy := &mockProxy{}
@@ -177,7 +179,7 @@ func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, PinTracke
 
 	tracer := &mockTracer{}
 
-	store := makeStore(t, badgerCfg, levelDBCfg)
+	store := makeStore(t, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg)
 	cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, false, crdtCfg)
 	tracker := stateless.New(statelesstrackerCfg, ident.ID, clusterCfg.Peername, cons.State)
 
@@ -229,27 +231,33 @@ func cleanState() {
 	os.RemoveAll(testsFolder)
 }
 
+func shutdownTestingCluster(ctx context.Context, t *testing.T, cl *Cluster) {
+	t.Helper()
+	err := cl.Shutdown(ctx)
+	if err != nil {
+		t.Fatal("cluster shutdown failed:", err)
+	}
+	cl.dht.Close()
+	cl.host.Close()
+	cl.datastore.Close()
+}
+
 func TestClusterShutdown(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
-	err := cl.Shutdown(ctx)
+	shutdownTestingCluster(ctx, t, cl)
-	if err != nil {
+	shutdownTestingCluster(ctx, t, cl)
-		t.Error("cluster shutdown failed:", err)
+
-	}
-	cl.Shutdown(ctx)
 	cl, _, _, _ = testingCluster(t)
-	err = cl.Shutdown(ctx)
+	shutdownTestingCluster(ctx, t, cl)
-	if err != nil {
+	cleanState()
-		t.Error("cluster shutdown failed:", err)
-	}
 }
 
 func TestClusterStateSync(t *testing.T) {
 	ctx := context.Background()
-	cleanState()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	c := test.Cid1
 	_, err := cl.Pin(ctx, c, api.PinOptions{})
@@ -279,7 +287,7 @@ func TestClusterID(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 	id := cl.ID(ctx)
 	if len(id.Addresses) == 0 {
 		t.Error("expected more addresses")
@@ -299,7 +307,7 @@ func TestClusterPin(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	c := test.Cid1
 	res, err := cl.Pin(ctx, c, api.PinOptions{})
@@ -332,7 +340,7 @@ func TestPinExpired(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	c := test.Cid1
 	_, err := cl.Pin(ctx, c, api.PinOptions{
@@ -347,7 +355,7 @@ func TestClusterPinPath(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	pin, err := cl.PinPath(ctx, test.PathIPFS2, api.PinOptions{})
 	if err != nil {
@@ -368,7 +376,7 @@ func TestAddFile(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 	sth := test.NewShardingTestHelper()
 	defer sth.Clean(t)
 
@@ -428,7 +436,7 @@ func TestUnpinShard(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 	sth := test.NewShardingTestHelper()
 	defer sth.Clean(t)
 
@@ -794,7 +802,7 @@ func TestClusterPins(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	c := test.Cid1
 	_, err := cl.Pin(ctx, c, api.PinOptions{})
@@ -820,7 +828,7 @@ func TestClusterPinGet(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	c := test.Cid1
 	_, err := cl.Pin(ctx, c, api.PinOptions{})
@@ -846,7 +854,7 @@ func TestClusterUnpin(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	c := test.Cid1
 	// Unpin should error without pin being committed to state
@@ -881,7 +889,7 @@ func TestClusterUnpinPath(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	// Unpin should error without pin being committed to state
 	_, err := cl.UnpinPath(ctx, test.PathIPFS2)
@@ -911,7 +919,7 @@ func TestClusterPeers(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	out := make(chan api.ID, 10)
 	cl.Peers(ctx, out)
@@ -935,7 +943,7 @@ func TestVersion(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 	if cl.Version() != version.Version.String() {
 		t.Error("bad Version()")
 	}
@@ -945,7 +953,7 @@ func TestClusterRecoverAllLocal(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	_, err := cl.Pin(ctx, test.ErrorCid, api.PinOptions{})
 	if err != nil {
@@ -974,7 +982,7 @@ func TestClusterRepoGC(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	gRepoGC, err := cl.RepoGC(ctx)
 	if err != nil {
@@ -998,7 +1006,7 @@ func TestClusterRepoGCLocal(t *testing.T) {
 	ctx := context.Background()
 	cl, _, _, _ := testingCluster(t)
 	defer cleanState()
-	defer cl.Shutdown(ctx)
+	defer shutdownTestingCluster(ctx, t, cl)
 
 	repoGC, err := cl.RepoGCLocal(ctx)
 	if err != nil {

config_test.go

@@ -8,7 +8,9 @@ import (
 	"github.com/ipfs-cluster/ipfs-cluster/consensus/crdt"
 	"github.com/ipfs-cluster/ipfs-cluster/consensus/raft"
 	"github.com/ipfs-cluster/ipfs-cluster/datastore/badger"
+	"github.com/ipfs-cluster/ipfs-cluster/datastore/badger3"
 	"github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb"
+	"github.com/ipfs-cluster/ipfs-cluster/datastore/pebble"
 	"github.com/ipfs-cluster/ipfs-cluster/informer/disk"
 	"github.com/ipfs-cluster/ipfs-cluster/ipfsconn/ipfshttp"
 	"github.com/ipfs-cluster/ipfs-cluster/monitor/pubsubmon"
@@ -74,12 +76,89 @@ var testingBadgerCfg = []byte(`{
     }
 }`)
 
+var testingBadger3Cfg = []byte(`
+{
+      "gc_discard_ratio": 0.2,
+      "gc_interval": "0s",
+      "gc_sleep": "0s",
+      "badger_options": {
+        "dir": "",
+        "value_dir": "",
+        "sync_writes": false,
+        "num_versions_to_keep": 1,
+        "read_only": false,
+        "compression": 0,
+        "in_memory": false,
+        "metrics_enabled": true,
+        "num_goroutines": 8,
+        "mem_table_size": 1048576,
+        "base_table_size": 2097152,
+        "base_level_size": 10485760,
+        "level_size_multiplier": 10,
+        "table_size_multiplier": 2,
+        "max_levels": 7,
+        "v_log_percentile": 0,
+        "value_threshold": 100,
+        "num_memtables": 5,
+        "block_size": 4096,
+        "bloom_false_positive": 0.01,
+        "block_cache_size": 0,
+        "index_cache_size": 0,
+        "num_level_zero_tables": 5,
+        "num_level_zero_tables_stall": 15,
+        "value_log_file_size": 1073741823,
+        "value_log_max_entries": 1000000,
+        "num_compactors": 4,
+        "compact_l_0_on_close": false,
+        "lmax_compaction": false,
+        "zstd_compression_level": 1,
+        "verify_value_checksum": false,
+        "checksum_verification_mode": 0,
+        "detect_conflicts": false,
+        "namespace_offset": -1
+      }
+    }
+`)
+
 var testingLevelDBCfg = []byte(`{
     "folder": "leveldbFromTests",
     "leveldb_options": {
     }
 }`)
 
+var testingPebbleCfg = []byte(`
+{
+      "pebble_options": {
+        "bytes_per_sync": 524288,
+        "disable_wal": false,
+        "flush_delay_delete_range": 0,
+        "flush_delay_range_key": 0,
+        "flush_split_bytes": 4194304,
+        "format_major_version": 1,
+        "l0_compaction_file_threshold": 500,
+        "l0_compaction_threshold": 4,
+        "l0_stop_writes_threshold": 12,
+        "l_base_max_bytes": 67108864,
+        "levels": [
+          {
+            "block_restart_interval": 16,
+            "block_size": 4096,
+            "block_size_threshold": 90,
+            "Compression": 1,
+            "filter_type": 0,
+            "index_block_size": 8000,
+            "target_file_size": 2097152
+          }
+        ],
+        "max_open_files": 1000,
+        "mem_table_size": 1048576,
+        "mem_table_stop_writes_threshold": 2,
+        "read_only": false,
+        "wal_bytes_per_sync": 0
+      }
+    }
+`)
+
 var testingAPICfg = []byte(`{
     "http_listen_multiaddress": "/ip4/127.0.0.1/tcp/10002",
     "read_timeout": "0",
@@ -144,16 +223,18 @@ var testingTracerCfg = []byte(`{
     "service_name": "cluster-daemon"
 }`)
 
-func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *leveldb.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) {
+func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *badger3.Config, *leveldb.Config, *pebble.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) {
-	identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingEmptyConfigs()
+	identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingEmptyConfigs()
 	identity.LoadJSON(testingIdentity)
 	clusterCfg.LoadJSON(testingClusterCfg)
 	apiCfg.LoadJSON(testingAPICfg)
 	proxyCfg.LoadJSON(testingProxyCfg)
 	ipfsCfg.LoadJSON(testingIpfsCfg)
 	badgerCfg.LoadJSON(testingBadgerCfg)
-	raftCfg.LoadJSON(testingRaftCfg)
+	badger3Cfg.LoadJSON(testingBadger3Cfg)
 	levelDBCfg.LoadJSON(testingLevelDBCfg)
+	pebbleCfg.LoadJSON(testingPebbleCfg)
+	raftCfg.LoadJSON(testingRaftCfg)
 	crdtCfg.LoadJSON(testingCrdtCfg)
 	statelesstrkrCfg.LoadJSON(testingTrackerCfg)
 	pubsubmonCfg.LoadJSON(testingMonCfg)
@@ -161,25 +242,27 @@ func testingConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Confi
 	diskInfCfg.LoadJSON(testingDiskInfCfg)
 	tracingCfg.LoadJSON(testingTracerCfg)
 
-	return identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg
+	return identity, clusterCfg, apiCfg, proxyCfg, ipfsCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrkrCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg
 }
 
-func testingEmptyConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *leveldb.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) {
+func testingEmptyConfigs() (*config.Identity, *Config, *rest.Config, *ipfsproxy.Config, *ipfshttp.Config, *badger.Config, *badger3.Config, *leveldb.Config, *pebble.Config, *raft.Config, *crdt.Config, *stateless.Config, *pubsubmon.Config, *balanced.Config, *disk.Config, *observations.TracingConfig) {
 	identity := &config.Identity{}
 	clusterCfg := &Config{}
 	apiCfg := rest.NewConfig()
 	proxyCfg := &ipfsproxy.Config{}
 	ipfshttpCfg := &ipfshttp.Config{}
 	badgerCfg := &badger.Config{}
-	raftCfg := &raft.Config{}
+	badger3Cfg := &badger3.Config{}
 	levelDBCfg := &leveldb.Config{}
+	pebbleCfg := &pebble.Config{}
+	raftCfg := &raft.Config{}
 	crdtCfg := &crdt.Config{}
 	statelessCfg := &stateless.Config{}
 	pubsubmonCfg := &pubsubmon.Config{}
 	allocBalancedCfg := &balanced.Config{}
 	diskInfCfg := &disk.Config{}
 	tracingCfg := &observations.TracingConfig{}
-	return identity, clusterCfg, apiCfg, proxyCfg, ipfshttpCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelessCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg
+	return identity, clusterCfg, apiCfg, proxyCfg, ipfshttpCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelessCfg, pubsubmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg
 }
 
 // func TestConfigDefault(t *testing.T) {

ipfscluster_test.go

@@ -21,8 +21,10 @@ import (
 	"github.com/ipfs-cluster/ipfs-cluster/consensus/crdt"
 	"github.com/ipfs-cluster/ipfs-cluster/consensus/raft"
 	"github.com/ipfs-cluster/ipfs-cluster/datastore/badger"
+	"github.com/ipfs-cluster/ipfs-cluster/datastore/badger3"
 	"github.com/ipfs-cluster/ipfs-cluster/datastore/inmem"
 	"github.com/ipfs-cluster/ipfs-cluster/datastore/leveldb"
+	"github.com/ipfs-cluster/ipfs-cluster/datastore/pebble"
 	"github.com/ipfs-cluster/ipfs-cluster/informer/disk"
 	"github.com/ipfs-cluster/ipfs-cluster/ipfsconn/ipfshttp"
 	"github.com/ipfs-cluster/ipfs-cluster/monitor/pubsubmon"
@@ -174,7 +176,7 @@ func createComponents(
 
 	peername := fmt.Sprintf("peer_%d", i)
 
-	ident, clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, badgerCfg, levelDBCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingConfigs()
+	ident, clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg, raftCfg, crdtCfg, statelesstrackerCfg, psmonCfg, allocBalancedCfg, diskInfCfg, tracingCfg := testingConfigs()
 
 	ident.ID = host.ID()
 	ident.PrivateKey = host.Peerstore().PrivKey(host.ID())
@@ -192,7 +194,9 @@ func createComponents(
 	raftCfg.DataFolder = filepath.Join(testsFolder, host.ID().Pretty())
 
 	badgerCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger")
+	badger3Cfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "badger3")
 	levelDBCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "leveldb")
+	pebbleCfg.Folder = filepath.Join(testsFolder, host.ID().Pretty(), "pebble")
 
 	api, err := rest.NewAPI(ctx, apiCfg)
 	if err != nil {
@@ -218,7 +222,7 @@ func createComponents(
 		t.Fatal(err)
 	}
 
-	store := makeStore(t, badgerCfg, levelDBCfg)
+	store := makeStore(t, badgerCfg, badger3Cfg, levelDBCfg, pebbleCfg)
 	cons := makeConsensus(t, store, host, pubsub, dht, raftCfg, staging, crdtCfg)
 	tracker := stateless.New(statelesstrackerCfg, ident.ID, clusterCfg.Peername, cons.State)
 
@@ -239,21 +243,39 @@ func createComponents(
 	return clusterCfg, store, cons, []API{api, ipfsProxy}, ipfs, tracker, mon, alloc, inf, tracer, mock
 }
 
-func makeStore(t *testing.T, badgerCfg *badger.Config, levelDBCfg *leveldb.Config) ds.Datastore {
+func makeStore(t *testing.T, badgerCfg *badger.Config, badger3Cfg *badger3.Config, levelDBCfg *leveldb.Config, pebbleCfg *pebble.Config) ds.Datastore {
 	switch consensus {
 	case "crdt":
-		if datastore == "badger" {
+		switch datastore {
+		case "badger":
 			dstr, err := badger.New(badgerCfg)
 			if err != nil {
 				t.Fatal(err)
 			}
 			return dstr
+		case "badger3":
+			dstr, err := badger3.New(badger3Cfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return dstr
+		case "leveldb":
+			dstr, err := leveldb.New(levelDBCfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return dstr
+		case "pebble":
+			dstr, err := pebble.New(pebbleCfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+			return dstr
+		default:
+			t.Fatal("bad datastore")
+			return nil
 		}
-		dstr, err := leveldb.New(levelDBCfg)
+
-		if err != nil {
-			t.Fatal(err)
-		}
-		return dstr
 	default:
 		return inmem.New()
 	}