5452b59a2e
* Update go-libp2p to v0.22.0 * Testing with go1.19 * build(deps): bump github.com/multiformats/go-multicodec Bumps [github.com/multiformats/go-multicodec](https://github.com/multiformats/go-multicodec) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/multiformats/go-multicodec/releases) - [Commits](https://github.com/multiformats/go-multicodec/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: github.com/multiformats/go-multicodec dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipld/go-car from 0.4.0 to 0.5.0 Bumps [github.com/ipld/go-car](https://github.com/ipld/go-car) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/ipld/go-car/releases) - [Commits](https://github.com/ipld/go-car/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-car dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/prometheus/client_golang Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.12.2 to 1.13.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.12.2...v1.13.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/hashicorp/go-hclog from 1.2.1 to 1.3.0 Bumps [github.com/hashicorp/go-hclog](https://github.com/hashicorp/go-hclog) from 1.2.1 to 1.3.0. - [Release notes](https://github.com/hashicorp/go-hclog/releases) - [Commits](https://github.com/hashicorp/go-hclog/compare/v1.2.1...v1.3.0) --- updated-dependencies: - dependency-name: github.com/hashicorp/go-hclog dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-ds-crdt from 0.3.6 to 0.3.7 Bumps [github.com/ipfs/go-ds-crdt](https://github.com/ipfs/go-ds-crdt) from 0.3.6 to 0.3.7. - [Release notes](https://github.com/ipfs/go-ds-crdt/releases) - [Commits](https://github.com/ipfs/go-ds-crdt/compare/v0.3.6...v0.3.7) --- updated-dependencies: - dependency-name: github.com/ipfs/go-ds-crdt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/urfave/cli/v2 from 2.10.2 to 2.14.1 Bumps [github.com/urfave/cli/v2](https://github.com/urfave/cli) from 2.10.2 to 2.14.1. - [Release notes](https://github.com/urfave/cli/releases) - [Changelog](https://github.com/urfave/cli/blob/main/docs/CHANGELOG.md) - [Commits](https://github.com/urfave/cli/compare/v2.10.2...v2.14.1) --- updated-dependencies: - dependency-name: github.com/urfave/cli/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/libp2p/go-libp2p-http from 0.3.0 to 0.4.0 Bumps [github.com/libp2p/go-libp2p-http](https://github.com/libp2p/go-libp2p-http) from 0.3.0 to 0.4.0. - [Release notes](https://github.com/libp2p/go-libp2p-http/releases) - [Commits](https://github.com/libp2p/go-libp2p-http/compare/v0.3.0...v0.4.0) --- updated-dependencies: - dependency-name: github.com/libp2p/go-libp2p-http dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/libp2p/go-libp2p-gorpc from 0.4.0 to 0.5.0 Bumps [github.com/libp2p/go-libp2p-gorpc](https://github.com/libp2p/go-libp2p-gorpc) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/libp2p/go-libp2p-gorpc/releases) - [Commits](https://github.com/libp2p/go-libp2p-gorpc/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: github.com/libp2p/go-libp2p-gorpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump contrib.go.opencensus.io/exporter/prometheus Bumps [contrib.go.opencensus.io/exporter/prometheus](https://github.com/census-ecosystem/opencensus-go-exporter-prometheus) from 0.4.1 to 0.4.2. - [Release notes](https://github.com/census-ecosystem/opencensus-go-exporter-prometheus/releases) - [Commits](https://github.com/census-ecosystem/opencensus-go-exporter-prometheus/compare/v0.4.1...v0.4.2) --- updated-dependencies: - dependency-name: contrib.go.opencensus.io/exporter/prometheus dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/libp2p/go-libp2p-raft from 0.1.8 to 0.2.0 Bumps [github.com/libp2p/go-libp2p-raft](https://github.com/libp2p/go-libp2p-raft) from 0.1.8 to 0.2.0. - [Release notes](https://github.com/libp2p/go-libp2p-raft/releases) - [Commits](https://github.com/libp2p/go-libp2p-raft/compare/v0.1.8...v0.2.0) --- updated-dependencies: - dependency-name: github.com/libp2p/go-libp2p-raft dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/urfave/cli from 1.22.9 to 1.22.10 Bumps [github.com/urfave/cli](https://github.com/urfave/cli) from 1.22.9 to 1.22.10. - [Release notes](https://github.com/urfave/cli/releases) - [Changelog](https://github.com/urfave/cli/blob/main/docs/CHANGELOG.md) - [Commits](https://github.com/urfave/cli/compare/v1.22.9...v1.22.10) --- updated-dependencies: - dependency-name: github.com/urfave/cli dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * Fix checker/linter/staticcheck warnings Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
272 lines
6.0 KiB
Go
272 lines
6.0 KiB
Go
package sharding
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"mime/multipart"
|
|
"sync"
|
|
"testing"
|
|
|
|
adder "github.com/ipfs-cluster/ipfs-cluster/adder"
|
|
"github.com/ipfs-cluster/ipfs-cluster/api"
|
|
"github.com/ipfs-cluster/ipfs-cluster/test"
|
|
|
|
logging "github.com/ipfs/go-log/v2"
|
|
peer "github.com/libp2p/go-libp2p/core/peer"
|
|
rpc "github.com/libp2p/go-libp2p-gorpc"
|
|
)
|
|
|
|
func init() {
|
|
logging.SetLogLevel("shardingdags", "INFO")
|
|
logging.SetLogLevel("adder", "INFO")
|
|
}
|
|
|
|
type testRPC struct {
|
|
blocks sync.Map
|
|
pins sync.Map
|
|
}
|
|
|
|
func (rpcs *testRPC) BlockStream(ctx context.Context, in <-chan api.NodeWithMeta, out chan<- struct{}) error {
|
|
defer close(out)
|
|
for n := range in {
|
|
rpcs.blocks.Store(n.Cid.String(), n.Data)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (rpcs *testRPC) Pin(ctx context.Context, in api.Pin, out *api.Pin) error {
|
|
rpcs.pins.Store(in.Cid.String(), in)
|
|
*out = in
|
|
return nil
|
|
}
|
|
|
|
func (rpcs *testRPC) BlockAllocate(ctx context.Context, in api.Pin, out *[]peer.ID) error {
|
|
if in.ReplicationFactorMin > 1 {
|
|
return errors.New("we can only replicate to 1 peer")
|
|
}
|
|
// it does not matter since we use host == nil for RPC, so it uses the
|
|
// local one in all cases
|
|
*out = []peer.ID{test.PeerID1}
|
|
return nil
|
|
}
|
|
|
|
func (rpcs *testRPC) PinGet(ctx context.Context, c api.Cid) (api.Pin, error) {
|
|
pI, ok := rpcs.pins.Load(c.String())
|
|
if !ok {
|
|
return api.Pin{}, errors.New("not found")
|
|
}
|
|
return pI.(api.Pin), nil
|
|
}
|
|
|
|
func (rpcs *testRPC) BlockGet(ctx context.Context, c api.Cid) ([]byte, error) {
|
|
bI, ok := rpcs.blocks.Load(c.String())
|
|
if !ok {
|
|
return nil, errors.New("not found")
|
|
}
|
|
return bI.([]byte), nil
|
|
}
|
|
|
|
func makeAdder(t *testing.T, params api.AddParams) (*adder.Adder, *testRPC) {
|
|
rpcObj := &testRPC{}
|
|
server := rpc.NewServer(nil, "mock")
|
|
err := server.RegisterName("Cluster", rpcObj)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = server.RegisterName("IPFSConnector", rpcObj)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client := rpc.NewClientWithServer(nil, "mock", server)
|
|
|
|
out := make(chan api.AddedOutput, 1)
|
|
|
|
dags := New(context.Background(), client, params, out)
|
|
add := adder.New(dags, params, out)
|
|
|
|
go func() {
|
|
for v := range out {
|
|
t.Logf("Output: Name: %s. Cid: %s. Size: %d", v.Name, v.Cid, v.Size)
|
|
}
|
|
}()
|
|
|
|
return add, rpcObj
|
|
}
|
|
|
|
func TestFromMultipart(t *testing.T) {
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
t.Run("Test tree", func(t *testing.T) {
|
|
p := api.DefaultAddParams()
|
|
// Total data is about
|
|
p.ShardSize = 1024 * 300 // 300kB
|
|
p.Name = "testingFile"
|
|
p.Shard = true
|
|
p.ReplicationFactorMin = 1
|
|
p.ReplicationFactorMax = 2
|
|
|
|
add, rpcObj := makeAdder(t, p)
|
|
_ = rpcObj
|
|
|
|
mr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
rootCid, err := add.FromMultipart(context.Background(), r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Print all pins
|
|
// rpcObj.pins.Range(func(k, v interface{}) bool {
|
|
// p := v.(*api.Pin)
|
|
// j, _ := config.DefaultJSONMarshal(p)
|
|
// fmt.Printf("%s", j)
|
|
// return true
|
|
// })
|
|
|
|
if rootCid.String() != test.ShardingDirBalancedRootCID {
|
|
t.Fatal("bad root CID")
|
|
}
|
|
|
|
// 14 has been obtained by carefully observing the logs
|
|
// making sure that splitting happens in the right place.
|
|
shardBlocks, err := VerifyShards(t, rootCid, rpcObj, rpcObj, 14)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, ci := range test.ShardingDirCids {
|
|
_, ok := shardBlocks[ci]
|
|
if !ok {
|
|
t.Fatal("shards are missing a block:", ci)
|
|
}
|
|
}
|
|
|
|
if len(test.ShardingDirCids) != len(shardBlocks) {
|
|
t.Fatal("shards have some extra blocks")
|
|
}
|
|
for _, ci := range test.ShardingDirCids {
|
|
_, ok := shardBlocks[ci]
|
|
if !ok {
|
|
t.Fatal("shards are missing a block:", ci)
|
|
}
|
|
}
|
|
|
|
if len(test.ShardingDirCids) != len(shardBlocks) {
|
|
t.Fatal("shards have some extra blocks")
|
|
}
|
|
|
|
})
|
|
|
|
t.Run("Test file", func(t *testing.T) {
|
|
p := api.DefaultAddParams()
|
|
// Total data is about
|
|
p.ShardSize = 1024 * 1024 * 2 // 2MB
|
|
p.Name = "testingFile"
|
|
p.Shard = true
|
|
p.ReplicationFactorMin = 1
|
|
p.ReplicationFactorMax = 2
|
|
|
|
add, rpcObj := makeAdder(t, p)
|
|
_ = rpcObj
|
|
|
|
mr, closer := sth.GetRandFileMultiReader(t, 1024*50) // 50 MB
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
rootCid, err := add.FromMultipart(context.Background(), r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
shardBlocks, err := VerifyShards(t, rootCid, rpcObj, rpcObj, 29)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = shardBlocks
|
|
})
|
|
|
|
}
|
|
|
|
func TestFromMultipart_Errors(t *testing.T) {
|
|
type testcase struct {
|
|
name string
|
|
params api.AddParams
|
|
}
|
|
|
|
tcs := []*testcase{
|
|
{
|
|
name: "bad chunker",
|
|
params: api.AddParams{
|
|
Format: "",
|
|
IPFSAddParams: api.IPFSAddParams{
|
|
Chunker: "aweee",
|
|
RawLeaves: false,
|
|
},
|
|
Hidden: false,
|
|
Shard: true,
|
|
PinOptions: api.PinOptions{
|
|
ReplicationFactorMin: -1,
|
|
ReplicationFactorMax: -1,
|
|
Name: "test",
|
|
ShardSize: 1024 * 1024,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "shard size too small",
|
|
params: api.AddParams{
|
|
Format: "",
|
|
IPFSAddParams: api.IPFSAddParams{
|
|
Chunker: "",
|
|
RawLeaves: false,
|
|
},
|
|
Hidden: false,
|
|
Shard: true,
|
|
PinOptions: api.PinOptions{
|
|
ReplicationFactorMin: -1,
|
|
ReplicationFactorMax: -1,
|
|
Name: "test",
|
|
ShardSize: 200,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "replication too high",
|
|
params: api.AddParams{
|
|
Format: "",
|
|
IPFSAddParams: api.IPFSAddParams{
|
|
Chunker: "",
|
|
RawLeaves: false,
|
|
},
|
|
Hidden: false,
|
|
Shard: true,
|
|
PinOptions: api.PinOptions{
|
|
ReplicationFactorMin: 2,
|
|
ReplicationFactorMax: 3,
|
|
Name: "test",
|
|
ShardSize: 1024 * 1024,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
for _, tc := range tcs {
|
|
add, rpcObj := makeAdder(t, tc.params)
|
|
_ = rpcObj
|
|
|
|
f := sth.GetTreeSerialFile(t)
|
|
|
|
_, err := add.FromFiles(context.Background(), f)
|
|
if err == nil {
|
|
t.Error(tc.name, ": expected an error")
|
|
} else {
|
|
t.Log(tc.name, ":", err)
|
|
}
|
|
f.Close()
|
|
}
|
|
}
|