9b9d76f92d
This commit introduces the new go-libp2p-gorpc streaming capabilities for Cluster. The main aim is to work towards heavily reducing memory usage when working with very large pinsets. As a side-effect, it takes the chance to revampt all types for all public methods so that pointers to static what should be static objects are not used anymore. This should heavily reduce heap allocations and GC activity. The main change is that state.List now returns a channel from which to read the pins, rather than pins being all loaded into a huge slice. Things reading pins have been all updated to iterate on the channel rather than on the slice. The full pinset is no longer fully loaded onto memory for things that run regularly like StateSync(). Additionally, the /allocations endpoint of the rest API no longer returns an array of pins, but rather streams json-encoded pin objects directly. This change has extended to the restapi client (which puts pins into a channel as they arrive) and to ipfs-cluster-ctl. There are still pending improvements like StatusAll() calls which should also stream responses, and specially BlockPut calls which should stream blocks directly into IPFS on a single call. These are coming up in future commits.
299 lines
7.4 KiB
Go
299 lines
7.4 KiB
Go
package ipfscluster
|
|
|
|
// This files has tests for Add* using multiple cluster peers.
|
|
|
|
import (
|
|
"context"
|
|
"mime/multipart"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
files "github.com/ipfs/go-ipfs-files"
|
|
"github.com/ipfs/ipfs-cluster/adder"
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
"github.com/ipfs/ipfs-cluster/test"
|
|
peer "github.com/libp2p/go-libp2p-core/peer"
|
|
)
|
|
|
|
func TestAdd(t *testing.T) {
|
|
ctx := context.Background()
|
|
clusters, mock := createClusters(t)
|
|
defer shutdownClusters(t, clusters, mock)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
|
|
|
t.Run("default", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
ci, err := clusters[0].AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ci.String() != test.ShardingDirBalancedRootCID {
|
|
t.Fatal("unexpected root CID for local add")
|
|
}
|
|
|
|
// We need to sleep a lot because it takes time to
|
|
// catch up on a first/single pin on crdts
|
|
time.Sleep(10 * time.Second)
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
pin := c.StatusLocal(ctx, ci)
|
|
if pin.Error != "" {
|
|
t.Error(pin.Error)
|
|
}
|
|
if pin.Status != api.TrackerStatusPinned {
|
|
t.Error("item should be pinned and is", pin.Status)
|
|
}
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
})
|
|
|
|
t.Run("local_one_allocation", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
params.ReplicationFactorMin = 1
|
|
params.ReplicationFactorMax = 1
|
|
params.Local = true
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
ci, err := clusters[2].AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ci.String() != test.ShardingDirBalancedRootCID {
|
|
t.Fatal("unexpected root CID for local add")
|
|
}
|
|
|
|
// We need to sleep a lot because it takes time to
|
|
// catch up on a first/single pin on crdts
|
|
time.Sleep(10 * time.Second)
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
pin := c.StatusLocal(ctx, ci)
|
|
if pin.Error != "" {
|
|
t.Error(pin.Error)
|
|
}
|
|
switch c.id {
|
|
case clusters[2].id:
|
|
if pin.Status != api.TrackerStatusPinned {
|
|
t.Error("item should be pinned and is", pin.Status)
|
|
}
|
|
default:
|
|
if pin.Status != api.TrackerStatusRemote {
|
|
t.Errorf("item should only be allocated to cluster2")
|
|
}
|
|
}
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
})
|
|
}
|
|
|
|
func TestAddWithUserAllocations(t *testing.T) {
|
|
ctx := context.Background()
|
|
clusters, mock := createClusters(t)
|
|
defer shutdownClusters(t, clusters, mock)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
|
|
|
t.Run("local", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.ReplicationFactorMin = 2
|
|
params.ReplicationFactorMax = 2
|
|
params.UserAllocations = []peer.ID{clusters[0].id, clusters[1].id}
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
ci, err := clusters[0].AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
pinDelay()
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
if c == clusters[0] || c == clusters[1] {
|
|
pin := c.StatusLocal(ctx, ci)
|
|
if pin.Error != "" {
|
|
t.Error(pin.Error)
|
|
}
|
|
if pin.Status != api.TrackerStatusPinned {
|
|
t.Error("item should be pinned and is", pin.Status)
|
|
}
|
|
} else {
|
|
pin := c.StatusLocal(ctx, ci)
|
|
if pin.Status != api.TrackerStatusRemote {
|
|
t.Error("expected tracker status remote")
|
|
}
|
|
}
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
})
|
|
}
|
|
|
|
func TestAddPeerDown(t *testing.T) {
|
|
ctx := context.Background()
|
|
clusters, mock := createClusters(t)
|
|
defer shutdownClusters(t, clusters, mock)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
err := clusters[0].Shutdown(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
|
|
|
t.Run("local", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
ci, err := clusters[1].AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if ci.String() != test.ShardingDirBalancedRootCID {
|
|
t.Fatal("unexpected root CID for local add")
|
|
}
|
|
|
|
// We need to sleep a lot because it takes time to
|
|
// catch up on a first/single pin on crdts
|
|
time.Sleep(10 * time.Second)
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
if c.id == clusters[0].id {
|
|
return
|
|
}
|
|
pin := c.StatusLocal(ctx, ci)
|
|
if pin.Error != "" {
|
|
t.Error(pin.Error)
|
|
}
|
|
if pin.Status != api.TrackerStatusPinned {
|
|
t.Error("item should be pinned")
|
|
}
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
})
|
|
}
|
|
|
|
func TestAddOnePeerFails(t *testing.T) {
|
|
clusters, mock := createClusters(t)
|
|
defer shutdownClusters(t, clusters, mock)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
|
|
|
t.Run("local", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
lg, closer := sth.GetRandFileReader(t, 100000) // 100 MB
|
|
defer closer.Close()
|
|
|
|
mr := files.NewMultiFileReader(lg, true)
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, err := clusters[0].AddFile(r, params)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
|
|
// Disconnect 1 cluster (the last). Things should keep working.
|
|
// Important that we close the hosts, otherwise the RPC
|
|
// Servers keep working along with BlockPuts.
|
|
time.Sleep(100 * time.Millisecond)
|
|
c := clusters[nClusters-1]
|
|
c.Shutdown(context.Background())
|
|
c.dht.Close()
|
|
c.host.Close()
|
|
wg.Wait()
|
|
})
|
|
}
|
|
|
|
func TestAddAllPeersFail(t *testing.T) {
|
|
ctx := context.Background()
|
|
clusters, mock := createClusters(t)
|
|
defer shutdownClusters(t, clusters, mock)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
|
|
|
t.Run("local", func(t *testing.T) {
|
|
// Prevent added content to be allocated to cluster 0
|
|
// as it is already going to have something.
|
|
_, err := clusters[0].Pin(ctx, test.Cid1, api.PinOptions{
|
|
ReplicationFactorMin: 1,
|
|
ReplicationFactorMax: 1,
|
|
UserAllocations: []peer.ID{clusters[0].host.ID()},
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ttlDelay()
|
|
|
|
params := api.DefaultAddParams()
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
// Allocate to every peer except 0 (which already has a pin)
|
|
params.PinOptions.ReplicationFactorMax = nClusters - 1
|
|
params.PinOptions.ReplicationFactorMin = nClusters - 1
|
|
|
|
lg, closer := sth.GetRandFileReader(t, 100000) // 100 MB
|
|
defer closer.Close()
|
|
mr := files.NewMultiFileReader(lg, true)
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
// var cid cid.Cid
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, err := clusters[0].AddFile(r, params)
|
|
if err != adder.ErrBlockAdder {
|
|
t.Error("expected ErrBlockAdder. Got: ", err)
|
|
}
|
|
}()
|
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
// Shutdown all clusters except 0 to see the right error.
|
|
// Important that we shut down the hosts, otherwise
|
|
// the RPC Servers keep working along with BlockPuts.
|
|
// Note that this kills raft.
|
|
runF(t, clusters[1:], func(t *testing.T, c *Cluster) {
|
|
c.Shutdown(ctx)
|
|
c.dht.Close()
|
|
c.host.Close()
|
|
})
|
|
wg.Wait()
|
|
})
|
|
}
|