cascade/ipfs-cluster
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix #1691: Error when adding directories with many files

Browse Source 
Fixes #1691 by updating to the latest go-unixfs and adding a test.

The test is verified to fail on the previous go-unixfs version.
This commit is contained in:
Hector Sanjuan 2022-06-15 20:49:11 +02:00
parent 144c66359e
commit 455aa6def1
4 changed files with 45 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
adder/adder_test.go
View File

@ -3,6 +3,7 @@ package adder
import (
"bytes"
"context"
"fmt"
"mime/multipart"
"sync"
"testing"
@ -23,7 +24,14 @@ type mockCDAGServ struct {
func newMockCDAGServ() *mockCDAGServ {
return &mockCDAGServ{
MockDAGService: test.NewMockDAGService(),
// write-only DAGs.
MockDAGService: test.NewMockDAGService(true),
}
}
func newReadableMockCDAGServ() *mockCDAGServ {
return &mockCDAGServ{
MockDAGService: test.NewMockDAGService(false),
}
}
@ -145,13 +153,14 @@ func TestAdder_CAR(t *testing.T) {
defer closer.Close()
r := multipart.NewReader(mr, mr.Boundary())
p := api.DefaultAddParams()
dags := newMockCDAGServ()
dags := newReadableMockCDAGServ()
adder := New(dags, p, nil)
root, err := adder.FromMultipart(ctx, r)
if err != nil {
t.Fatal(err)
}
var carBuf bytes.Buffer
// Make a CAR out of the files we added.
err = car.WriteCar(ctx, dags, []cid.Cid{root.Cid}, &carBuf)
if err != nil {
t.Fatal(err)
@ -188,3 +197,31 @@ func TestAdder_CAR(t *testing.T) {
}
}
func TestAdder_LargeFolder(t *testing.T) {
items := 10000 // add 10000 items
sth := test.NewShardingTestHelper()
defer sth.Clean(t)
filesMap := make(map[string]files.Node)
for i := 0; i < items; i++ {
fstr := fmt.Sprintf("file%d", i)
f := files.NewBytesFile([]byte(fstr))
filesMap[fstr] = f
}
slf := files.NewMapDirectory(filesMap)
p := api.DefaultAddParams()
p.Wrap = true
dags := newMockCDAGServ()
adder := New(dags, p, nil)
_, err := adder.FromFiles(context.Background(), slf)
if err != nil {
t.Fatal(err)
}
}

3
adder/util.go
View File

@ -3,7 +3,6 @@ package adder
import (
"context"
"errors"
"fmt"
"sync"
"github.com/ipfs-cluster/ipfs-cluster/api"
@ -165,7 +164,7 @@ func (dag BaseDAGService) Get(ctx context.Context, key cid.Cid) (ipld.Node, erro
// GetMany returns an output channel that always emits an error
func (dag BaseDAGService) GetMany(ctx context.Context, keys []cid.Cid) <-chan *ipld.NodeOption {
out := make(chan *ipld.NodeOption, 1)
out <- &ipld.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
out <- &ipld.NodeOption{Err: errors.New("failed to fetch all nodes")}
close(out)
return out
}

2
go.mod
View File

@ -36,7 +36,7 @@ require (
github.com/ipfs/go-merkledag v0.6.0
github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164
github.com/ipfs/go-path v0.3.0
github.com/ipfs/go-unixfs v0.3.1
github.com/ipfs/go-unixfs v0.4.0
github.com/ipld/go-car v0.3.3
github.com/kelseyhightower/envconfig v1.4.0
github.com/kishansagathiya/go-dot v0.1.0

4
go.sum
View File

@ -602,6 +602,10 @@ github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMU
github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8=
github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o=
github.com/ipfs/go-unixfs v0.3.2-0.20220615181143-9ac1b9ff0917 h1:TzNOzSioqAfd6VQS8+PWAnKH4M7shjyDVe6iSvQO7HE=
github.com/ipfs/go-unixfs v0.3.2-0.20220615181143-9ac1b9ff0917/go.mod h1:I7Nqtm06HgOOd+setAoCU6rf/HgVFHE+peeNuOv/5+g=
github.com/ipfs/go-unixfs v0.4.0 h1:qSyyxfB/OiDdWHYiSbyaqKC7zfSE/TFL0QdwkRjBm20=
github.com/ipfs/go-unixfs v0.4.0/go.mod h1:I7Nqtm06HgOOd+setAoCU6rf/HgVFHE+peeNuOv/5+g=
github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=