9b9d76f92d
This commit introduces the new go-libp2p-gorpc streaming capabilities for Cluster. The main aim is to work towards heavily reducing memory usage when working with very large pinsets. As a side-effect, it takes the chance to revampt all types for all public methods so that pointers to static what should be static objects are not used anymore. This should heavily reduce heap allocations and GC activity. The main change is that state.List now returns a channel from which to read the pins, rather than pins being all loaded into a huge slice. Things reading pins have been all updated to iterate on the channel rather than on the slice. The full pinset is no longer fully loaded onto memory for things that run regularly like StateSync(). Additionally, the /allocations endpoint of the rest API no longer returns an array of pins, but rather streams json-encoded pin objects directly. This change has extended to the restapi client (which puts pins into a channel as they arrive) and to ipfs-cluster-ctl. There are still pending improvements like StatusAll() calls which should also stream responses, and specially BlockPut calls which should stream blocks directly into IPFS on a single call. These are coming up in future commits.
109 lines
2.8 KiB
Go
109 lines
2.8 KiB
Go
package sharding
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
cid "github.com/ipfs/go-cid"
|
|
)
|
|
|
|
// MockPinStore is used in VerifyShards
|
|
type MockPinStore interface {
|
|
// Gets a pin
|
|
PinGet(context.Context, cid.Cid) (api.Pin, error)
|
|
}
|
|
|
|
// MockBlockStore is used in VerifyShards
|
|
type MockBlockStore interface {
|
|
// Gets a block
|
|
BlockGet(context.Context, cid.Cid) ([]byte, error)
|
|
}
|
|
|
|
// VerifyShards checks that a sharded CID has been correctly formed and stored.
|
|
// This is a helper function for testing. It returns a map with all the blocks
|
|
// from all shards.
|
|
func VerifyShards(t *testing.T, rootCid cid.Cid, pins MockPinStore, ipfs MockBlockStore, expectedShards int) (map[string]struct{}, error) {
|
|
ctx := context.Background()
|
|
metaPin, err := pins.PinGet(ctx, rootCid)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("meta pin was not pinned: %s", err)
|
|
}
|
|
|
|
if api.PinType(metaPin.Type) != api.MetaType {
|
|
return nil, fmt.Errorf("bad MetaPin type")
|
|
}
|
|
|
|
if metaPin.Reference == nil {
|
|
return nil, errors.New("metaPin.Reference is unset")
|
|
}
|
|
|
|
clusterPin, err := pins.PinGet(ctx, *metaPin.Reference)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cluster pin was not pinned: %s", err)
|
|
}
|
|
if api.PinType(clusterPin.Type) != api.ClusterDAGType {
|
|
return nil, fmt.Errorf("bad ClusterDAGPin type")
|
|
}
|
|
|
|
if !clusterPin.Reference.Equals(metaPin.Cid) {
|
|
return nil, fmt.Errorf("clusterDAG should reference the MetaPin")
|
|
}
|
|
|
|
clusterDAGBlock, err := ipfs.BlockGet(ctx, clusterPin.Cid)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("cluster pin was not stored: %s", err)
|
|
}
|
|
|
|
clusterDAGNode, err := CborDataToNode(clusterDAGBlock, "cbor")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
shards := clusterDAGNode.Links()
|
|
if len(shards) != expectedShards {
|
|
return nil, fmt.Errorf("bad number of shards")
|
|
}
|
|
|
|
shardBlocks := make(map[string]struct{})
|
|
var ref cid.Cid
|
|
// traverse shards in order
|
|
for i := 0; i < len(shards); i++ {
|
|
sh, _, err := clusterDAGNode.ResolveLink([]string{fmt.Sprintf("%d", i)})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
shardPin, err := pins.PinGet(ctx, sh.Cid)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("shard was not pinned: %s %s", sh.Cid, err)
|
|
}
|
|
|
|
if ref != cid.Undef && !shardPin.Reference.Equals(ref) {
|
|
t.Errorf("Ref (%s) should point to previous shard (%s)", ref, shardPin.Reference)
|
|
}
|
|
ref = shardPin.Cid
|
|
|
|
shardBlock, err := ipfs.BlockGet(ctx, shardPin.Cid)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("shard block was not stored: %s", err)
|
|
}
|
|
shardNode, err := CborDataToNode(shardBlock, "cbor")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, l := range shardNode.Links() {
|
|
ci := l.Cid.String()
|
|
_, ok := shardBlocks[ci]
|
|
if ok {
|
|
return nil, fmt.Errorf("block belongs to two shards: %s", ci)
|
|
}
|
|
shardBlocks[ci] = struct{}{}
|
|
}
|
|
}
|
|
return shardBlocks, nil
|
|
}
|