d553227755
* build(deps): bump github.com/hsanjuan/ipfs-lite from 1.5.0 to 1.6.0 Bumps [github.com/hsanjuan/ipfs-lite](https://github.com/hsanjuan/ipfs-lite) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/hsanjuan/ipfs-lite/releases) - [Commits](https://github.com/hsanjuan/ipfs-lite/compare/v1.5.0...v1.6.0) --- updated-dependencies: - dependency-name: github.com/hsanjuan/ipfs-lite dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/golang-jwt/jwt/v4 from 4.4.3 to 4.5.0 Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.4.3 to 4.5.0. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.4.3...v4.5.0) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * Update pubsub too * Fix compatiblity with latest libp2p, libipfs * build(deps): bump github.com/libp2p/go-libp2p-raft from 0.3.0 to 0.4.0 Bumps [github.com/libp2p/go-libp2p-raft](https://github.com/libp2p/go-libp2p-raft) from 0.3.0 to 0.4.0. - [Release notes](https://github.com/libp2p/go-libp2p-raft/releases) - [Commits](https://github.com/libp2p/go-libp2p-raft/compare/v0.3.0...v0.4.0) --- updated-dependencies: - dependency-name: github.com/libp2p/go-libp2p-raft dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/multiformats/go-multicodec Bumps [github.com/multiformats/go-multicodec](https://github.com/multiformats/go-multicodec) from 0.7.0 to 0.8.1. - [Release notes](https://github.com/multiformats/go-multicodec/releases) - [Commits](https://github.com/multiformats/go-multicodec/compare/v0.7.0...v0.8.1) --- updated-dependencies: - dependency-name: github.com/multiformats/go-multicodec dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipld/go-car from 0.5.0 to 0.6.0 Bumps [github.com/ipld/go-car](https://github.com/ipld/go-car) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/ipld/go-car/releases) - [Changelog](https://github.com/ipld/go-car/blob/master/.goreleaser.yaml) - [Commits](https://github.com/ipld/go-car/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: github.com/ipld/go-car dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump golang.org/x/crypto from 0.5.0 to 0.6.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/golang/crypto/releases) - [Commits](https://github.com/golang/crypto/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-ds-pebble from 0.1.0 to 0.2.2 Bumps [github.com/ipfs/go-ds-pebble](https://github.com/ipfs/go-ds-pebble) from 0.1.0 to 0.2.2. - [Release notes](https://github.com/ipfs/go-ds-pebble/releases) - [Commits](https://github.com/ipfs/go-ds-pebble/compare/v0.1.0...v0.2.2) --- updated-dependencies: - dependency-name: github.com/ipfs/go-ds-pebble dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ugorji/go/codec from 1.2.8 to 1.2.10 Bumps [github.com/ugorji/go/codec](https://github.com/ugorji/go) from 1.2.8 to 1.2.10. - [Release notes](https://github.com/ugorji/go/releases) - [Commits](https://github.com/ugorji/go/compare/v1.2.8...v1.2.10) --- updated-dependencies: - dependency-name: github.com/ugorji/go/codec dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/libp2p/go-libp2p-http from 0.4.0 to 0.5.0 Bumps [github.com/libp2p/go-libp2p-http](https://github.com/libp2p/go-libp2p-http) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/libp2p/go-libp2p-http/releases) - [Commits](https://github.com/libp2p/go-libp2p-http/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: github.com/libp2p/go-libp2p-http dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-ipfs-pinner from 0.2.1 to 0.3.0 Bumps [github.com/ipfs/go-ipfs-pinner](https://github.com/ipfs/go-ipfs-pinner) from 0.2.1 to 0.3.0. - [Release notes](https://github.com/ipfs/go-ipfs-pinner/releases) - [Commits](https://github.com/ipfs/go-ipfs-pinner/compare/v0.2.1...v0.3.0) --- updated-dependencies: - dependency-name: github.com/ipfs/go-ipfs-pinner dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-unixfs from 0.4.3 to 0.4.4 Bumps [github.com/ipfs/go-unixfs](https://github.com/ipfs/go-unixfs) from 0.4.3 to 0.4.4. - [Release notes](https://github.com/ipfs/go-unixfs/releases) - [Commits](https://github.com/ipfs/go-unixfs/compare/v0.4.3...v0.4.4) --- updated-dependencies: - dependency-name: github.com/ipfs/go-unixfs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-path from 0.3.0 to 0.3.1 Bumps [github.com/ipfs/go-path](https://github.com/ipfs/go-path) from 0.3.0 to 0.3.1. - [Release notes](https://github.com/ipfs/go-path/releases) - [Commits](https://github.com/ipfs/go-path/compare/v0.3.0...v0.3.1) --- updated-dependencies: - dependency-name: github.com/ipfs/go-path dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-ds-crdt from 0.3.9 to 0.3.10 Bumps [github.com/ipfs/go-ds-crdt](https://github.com/ipfs/go-ds-crdt) from 0.3.9 to 0.3.10. - [Release notes](https://github.com/ipfs/go-ds-crdt/releases) - [Commits](https://github.com/ipfs/go-ds-crdt/compare/v0.3.9...v0.3.10) --- updated-dependencies: - dependency-name: github.com/ipfs/go-ds-crdt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump golang.org/x/crypto from 0.6.0 to 0.7.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/golang/crypto/releases) - [Commits](https://github.com/golang/crypto/compare/v0.6.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> * build(deps): bump github.com/ipfs/go-ds-pebble from 0.2.2 to 0.2.3 Bumps [github.com/ipfs/go-ds-pebble](https://github.com/ipfs/go-ds-pebble) from 0.2.2 to 0.2.3. - [Release notes](https://github.com/ipfs/go-ds-pebble/releases) - [Commits](https://github.com/ipfs/go-ds-pebble/compare/v0.2.2...v0.2.3) --- updated-dependencies: - dependency-name: github.com/ipfs/go-ds-pebble dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> --------- Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
234 lines
4.8 KiB
Go
234 lines
4.8 KiB
Go
package adder
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"mime/multipart"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ipfs-cluster/ipfs-cluster/api"
|
|
"github.com/ipfs-cluster/ipfs-cluster/test"
|
|
"github.com/ipld/go-car"
|
|
peer "github.com/libp2p/go-libp2p/core/peer"
|
|
|
|
cid "github.com/ipfs/go-cid"
|
|
files "github.com/ipfs/go-libipfs/files"
|
|
)
|
|
|
|
type mockCDAGServ struct {
|
|
*test.MockDAGService
|
|
}
|
|
|
|
func newMockCDAGServ() *mockCDAGServ {
|
|
return &mockCDAGServ{
|
|
// write-only DAGs.
|
|
MockDAGService: test.NewMockDAGService(true),
|
|
}
|
|
}
|
|
|
|
func newReadableMockCDAGServ() *mockCDAGServ {
|
|
return &mockCDAGServ{
|
|
MockDAGService: test.NewMockDAGService(false),
|
|
}
|
|
}
|
|
|
|
// noop
|
|
func (dag *mockCDAGServ) Finalize(ctx context.Context, root api.Cid) (api.Cid, error) {
|
|
return root, nil
|
|
}
|
|
|
|
func (dag *mockCDAGServ) Allocations() []peer.ID {
|
|
return nil
|
|
}
|
|
|
|
func TestAdder(t *testing.T) {
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
mr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
p := api.DefaultAddParams()
|
|
expectedCids := test.ShardingDirCids[:]
|
|
|
|
dags := newMockCDAGServ()
|
|
defer dags.Close()
|
|
|
|
adder := New(dags, p, nil)
|
|
|
|
root, err := adder.FromMultipart(context.Background(), r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if root.String() != test.ShardingDirBalancedRootCID {
|
|
t.Error("expected the right content root")
|
|
}
|
|
|
|
if len(expectedCids) != len(dags.Nodes) {
|
|
t.Fatal("unexpected number of blocks imported")
|
|
}
|
|
|
|
for _, c := range expectedCids {
|
|
ci, _ := cid.Decode(c)
|
|
_, ok := dags.Nodes[ci]
|
|
if !ok {
|
|
t.Fatal("unexpected block emitted:", c)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdder_DoubleStart(t *testing.T) {
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
f := sth.GetTreeSerialFile(t)
|
|
p := api.DefaultAddParams()
|
|
|
|
dags := newMockCDAGServ()
|
|
defer dags.Close()
|
|
|
|
adder := New(dags, p, nil)
|
|
_, err := adder.FromFiles(context.Background(), f)
|
|
f.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
f = sth.GetTreeSerialFile(t)
|
|
_, err = adder.FromFiles(context.Background(), f)
|
|
f.Close()
|
|
if err == nil {
|
|
t.Fatal("expected an error: cannot run importer twice")
|
|
}
|
|
}
|
|
|
|
func TestAdder_ContextCancelled(t *testing.T) {
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
lg, closer := sth.GetRandFileReader(t, 100000) // 50 MB
|
|
st := sth.GetTreeSerialFile(t)
|
|
defer closer.Close()
|
|
defer st.Close()
|
|
|
|
slf := files.NewMapDirectory(map[string]files.Node{
|
|
"a": lg,
|
|
"b": st,
|
|
})
|
|
mr := files.NewMultiFileReader(slf, true)
|
|
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
p := api.DefaultAddParams()
|
|
|
|
dags := newMockCDAGServ()
|
|
defer dags.Close()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
adder := New(dags, p, nil)
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, err := adder.FromMultipart(ctx, r)
|
|
if err == nil {
|
|
t.Error("expected a context canceled error")
|
|
}
|
|
t.Log(err)
|
|
}()
|
|
// adder.FromMultipart will finish, if sleep more
|
|
time.Sleep(50 * time.Millisecond)
|
|
cancel()
|
|
wg.Wait()
|
|
}
|
|
|
|
func TestAdder_CAR(t *testing.T) {
|
|
// prepare a CAR file
|
|
ctx := context.Background()
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
mr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
p := api.DefaultAddParams()
|
|
dags := newReadableMockCDAGServ()
|
|
adder := New(dags, p, nil)
|
|
root, err := adder.FromMultipart(ctx, r)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var carBuf bytes.Buffer
|
|
// Make a CAR out of the files we added.
|
|
err = car.WriteCar(ctx, dags, []cid.Cid{root.Cid}, &carBuf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Make the CAR look like a multipart.
|
|
carFile := files.NewReaderFile(&carBuf)
|
|
carDir := files.NewMapDirectory(
|
|
map[string]files.Node{"": carFile},
|
|
)
|
|
carMf := files.NewMultiFileReader(carDir, true)
|
|
carMr := multipart.NewReader(carMf, carMf.Boundary())
|
|
|
|
// Add the car, discarding old dags.
|
|
dags = newMockCDAGServ()
|
|
defer dags.Close()
|
|
|
|
p.Format = "car"
|
|
adder = New(dags, p, nil)
|
|
root2, err := adder.FromMultipart(ctx, carMr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !root.Equals(root2) {
|
|
t.Error("Imported CAR file does not have expected root")
|
|
}
|
|
|
|
expectedCids := test.ShardingDirCids[:]
|
|
for _, c := range expectedCids {
|
|
ci, _ := cid.Decode(c)
|
|
_, ok := dags.Nodes[ci]
|
|
if !ok {
|
|
t.Fatal("unexpected block extracted from CAR:", c)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func TestAdder_LargeFolder(t *testing.T) {
|
|
items := 10000 // add 10000 items
|
|
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
filesMap := make(map[string]files.Node)
|
|
for i := 0; i < items; i++ {
|
|
fstr := fmt.Sprintf("file%d", i)
|
|
f := files.NewBytesFile([]byte(fstr))
|
|
filesMap[fstr] = f
|
|
}
|
|
|
|
slf := files.NewMapDirectory(filesMap)
|
|
|
|
p := api.DefaultAddParams()
|
|
p.Wrap = true
|
|
|
|
dags := newMockCDAGServ()
|
|
defer dags.Close()
|
|
|
|
adder := New(dags, p, nil)
|
|
_, err := adder.FromFiles(context.Background(), slf)
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|