2018-08-07 18:01:02 +00:00
|
|
|
package adder
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"mime/multipart"
|
|
|
|
"sync"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
"github.com/ipfs/ipfs-cluster/test"
|
|
|
|
|
2018-12-06 18:59:05 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
|
|
|
files "github.com/ipfs/go-ipfs-files"
|
|
|
|
ipld "github.com/ipfs/go-ipld-format"
|
2018-08-07 18:01:02 +00:00
|
|
|
)
|
|
|
|
|
2018-08-09 11:22:47 +00:00
|
|
|
type mockCDAGServ struct {
|
2018-08-07 18:01:02 +00:00
|
|
|
BaseDAGService
|
|
|
|
resultCids map[string]struct{}
|
|
|
|
}
|
|
|
|
|
2018-08-09 11:22:47 +00:00
|
|
|
func (dag *mockCDAGServ) Add(ctx context.Context, node ipld.Node) error {
|
2018-08-07 18:01:02 +00:00
|
|
|
dag.resultCids[node.Cid().String()] = struct{}{}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-08-09 11:22:47 +00:00
|
|
|
func (dag *mockCDAGServ) AddMany(ctx context.Context, nodes []ipld.Node) error {
|
2018-08-07 18:01:02 +00:00
|
|
|
for _, node := range nodes {
|
|
|
|
err := dag.Add(ctx, node)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-09-22 01:00:10 +00:00
|
|
|
func (dag *mockCDAGServ) Finalize(ctx context.Context, root cid.Cid) (cid.Cid, error) {
|
2018-08-08 23:16:30 +00:00
|
|
|
return root, nil
|
2018-08-07 18:01:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func TestAdder(t *testing.T) {
|
|
|
|
sth := test.NewShardingTestHelper()
|
2018-08-09 08:21:22 +00:00
|
|
|
defer sth.Clean(t)
|
2018-08-07 18:01:02 +00:00
|
|
|
|
2018-08-09 08:21:22 +00:00
|
|
|
mr, closer := sth.GetTreeMultiReader(t)
|
|
|
|
defer closer.Close()
|
2018-08-07 18:01:02 +00:00
|
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
p := api.DefaultAddParams()
|
|
|
|
expectedCids := test.ShardingDirCids[:]
|
|
|
|
|
2018-08-09 11:22:47 +00:00
|
|
|
dags := &mockCDAGServ{
|
2018-08-07 18:01:02 +00:00
|
|
|
resultCids: make(map[string]struct{}),
|
|
|
|
}
|
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
adder := New(dags, p, nil)
|
2018-08-07 18:01:02 +00:00
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
root, err := adder.FromMultipart(context.Background(), r)
|
2018-08-07 18:01:02 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if root.String() != test.ShardingDirBalancedRootCID {
|
|
|
|
t.Error("expected the right content root")
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(expectedCids) != len(dags.resultCids) {
|
|
|
|
t.Fatal("unexpected number of blocks imported")
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range expectedCids {
|
|
|
|
_, ok := dags.resultCids[c]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("unexpected block emitted:", c)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAdder_DoubleStart(t *testing.T) {
|
|
|
|
sth := test.NewShardingTestHelper()
|
2018-08-09 08:21:22 +00:00
|
|
|
defer sth.Clean(t)
|
2018-08-07 18:01:02 +00:00
|
|
|
|
|
|
|
f := sth.GetTreeSerialFile(t)
|
|
|
|
p := api.DefaultAddParams()
|
|
|
|
|
2018-08-09 11:22:47 +00:00
|
|
|
dags := &mockCDAGServ{
|
2018-08-07 18:01:02 +00:00
|
|
|
resultCids: make(map[string]struct{}),
|
|
|
|
}
|
|
|
|
|
2018-08-08 19:10:42 +00:00
|
|
|
adder := New(dags, p, nil)
|
2018-08-10 09:51:27 +00:00
|
|
|
_, err := adder.FromFiles(context.Background(), f)
|
2018-08-09 08:21:22 +00:00
|
|
|
f.Close()
|
2018-08-07 18:01:02 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
f = sth.GetTreeSerialFile(t)
|
2018-08-10 09:51:27 +00:00
|
|
|
_, err = adder.FromFiles(context.Background(), f)
|
2018-08-09 08:21:22 +00:00
|
|
|
f.Close()
|
2018-08-07 18:01:02 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Fatal("expected an error: cannot run importer twice")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAdder_ContextCancelled(t *testing.T) {
|
|
|
|
sth := test.NewShardingTestHelper()
|
2018-08-09 08:21:22 +00:00
|
|
|
defer sth.Clean(t)
|
2018-08-07 18:01:02 +00:00
|
|
|
|
2018-12-03 10:30:31 +00:00
|
|
|
lg, closer := sth.GetRandFileReader(t, 50000) // 50 MB
|
|
|
|
st := sth.GetTreeSerialFile(t)
|
2018-08-09 08:21:22 +00:00
|
|
|
defer closer.Close()
|
2018-12-03 10:30:31 +00:00
|
|
|
defer st.Close()
|
|
|
|
|
2018-12-17 12:44:06 +00:00
|
|
|
slf := files.NewMapDirectory(map[string]files.Node{
|
2018-12-03 15:13:11 +00:00
|
|
|
"a": lg,
|
|
|
|
"b": st,
|
2018-12-03 10:30:31 +00:00
|
|
|
})
|
2018-12-03 14:45:35 +00:00
|
|
|
mr := files.NewMultiFileReader(slf, true)
|
2018-12-03 10:30:31 +00:00
|
|
|
|
2018-08-07 18:01:02 +00:00
|
|
|
r := multipart.NewReader(mr, mr.Boundary())
|
|
|
|
|
|
|
|
p := api.DefaultAddParams()
|
|
|
|
|
2018-08-09 11:22:47 +00:00
|
|
|
dags := &mockCDAGServ{
|
2018-08-07 18:01:02 +00:00
|
|
|
resultCids: make(map[string]struct{}),
|
|
|
|
}
|
|
|
|
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2018-08-08 19:10:42 +00:00
|
|
|
adder := New(dags, p, nil)
|
2018-08-07 18:01:02 +00:00
|
|
|
var wg sync.WaitGroup
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
2018-08-08 19:10:42 +00:00
|
|
|
_, err := adder.FromMultipart(ctx, r)
|
2018-08-07 18:01:02 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Error("expected a context cancelled error")
|
|
|
|
}
|
|
|
|
t.Log(err)
|
|
|
|
}()
|
2019-08-09 15:13:01 +00:00
|
|
|
time.Sleep(100 * time.Millisecond)
|
2018-08-07 18:01:02 +00:00
|
|
|
cancel()
|
|
|
|
wg.Wait()
|
|
|
|
}
|