License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
This commit is contained in:
Hector Sanjuan 2018-07-17 13:35:23 +02:00
parent a96241941e
commit 9d89bda2a6
8 changed files with 279 additions and 187 deletions

1
.gitignore vendored
View File

@ -9,6 +9,7 @@ sharness/trash*
raftFolderFromTest*
peerstore
shardTesting
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o

View File

@ -100,24 +100,24 @@ func (dag *adderDAGService) RemoveMany(ctx context.Context, keys []*cid.Cid) err
return nil
}
// printDAGService will "add" a node by printing it. printDAGService cannot Get nodes
// that have already been seen and calls to Remove are noops. Nodes are
// recorded after being added so that they will only be printed once.
type printDAGService struct {
ads ipld.DAGService
}
// // printDAGService will "add" a node by printing it. printDAGService cannot Get nodes
// // that have already been seen and calls to Remove are noops. Nodes are
// // recorded after being added so that they will only be printed once.
// type printDAGService struct {
// ads ipld.DAGService
// }
func newPDagService() *printDAGService {
ch := make(chan *api.NodeWithMeta)
ads := newAdderDAGService(ch)
// func newPDagService() *printDAGService {
// ch := make(chan *api.NodeWithMeta)
// ads := newAdderDAGService(ch)
go func() {
for n := range ch {
fmt.Printf(n.Cid, " | ", n.Size)
}
}()
// go func() {
// for n := range ch {
// fmt.Printf(n.Cid, " | ", n.Size)
// }
// }()
return &printDAGService{
ads: ads,
}
}
// return &printDAGService{
// ads: ads,
// }
// }

View File

@ -108,26 +108,26 @@ func (imp *Importer) Go(ctx context.Context) error {
case <-ctx.Done():
imp.errors <- ctx.Err()
return
}
f, err := imp.files.NextFile()
if err != nil {
if err == io.EOF {
break // time to finalize
default:
f, err := imp.files.NextFile()
if err != nil {
if err == io.EOF {
goto FINALIZE // time to finalize
}
imp.errors <- err
return
}
imp.errors <- err
return
}
if err := ipfsAdder.AddFile(f); err != nil {
imp.errors <- err
return
if err := ipfsAdder.AddFile(f); err != nil {
imp.errors <- err
return
}
}
}
FINALIZE:
_, err := ipfsAdder.Finalize()
if err != nil {
if !isNotFound(err) {
if isNotFound(err) {
fmt.Println("fixme importer.go", err)
} else {
imp.errors <- err
@ -144,6 +144,7 @@ func (imp *Importer) Run(ctx context.Context, blockF BlockHandler) (string, erro
errors := imp.Errors()
blocks := imp.Blocks()
output := imp.Output()
err := imp.Go(ctx)
if err != nil {
@ -156,19 +157,22 @@ func (imp *Importer) Run(ctx context.Context, blockF BlockHandler) (string, erro
return retVal, ctx.Err()
case err, ok := <-errors:
if ok {
fmt.Println(err)
return retVal, err
}
case node, ok := <-blocks:
if !ok {
break // finished importing
goto BREAK // finished importing
}
retVal, err := blockF(ctx, node)
retVal, err = blockF(ctx, node)
if err != nil {
return retVal, err
}
case <-output:
// TODO
}
}
BREAK:
// grab any last errors from errors if necessary
// (this waits for errors to be closed)
err = <-errors

View File

@ -8,107 +8,69 @@ import (
"github.com/ipfs/ipfs-cluster/test"
)
// import and receive all blocks
func TestToChannelOutput(t *testing.T) {
file, err := test.GetTestingDirSerial()
func TestImporter(t *testing.T) {
f := test.GetTestingDirSerial(t)
p := DefaultParams()
imp, err := NewImporter(f, p)
if err != nil {
t.Fatal(err)
}
printChan, outChan, errChan := ToChannel(context.Background(), file,
false, false, false, "")
expectedCids := test.TestDirCids[:]
resultCids := make(map[string]struct{})
go func() { // listen on printChan so progress can be made
for {
_, ok := <-printChan
if !ok {
// channel closed, safe to exit
return
}
}
}()
go listenErrCh(t, errChan)
objs := make([]interface{}, 0)
for obj := range outChan {
objs = append(objs, obj)
blockHandler := func(ctx context.Context, n *api.NodeWithMeta) (string, error) {
resultCids[n.Cid] = struct{}{}
return n.Cid, nil
}
testChannelOutput(t, objs, test.TestDirCids[:])
}
func TestToChannelPrint(t *testing.T) {
file, err := test.GetTestingDirSerial()
_, err = imp.Run(context.Background(), blockHandler)
if err != nil {
t.Fatal(err)
}
printChan, outChan, errChan := ToChannel(context.Background(), file,
false, false, false, "")
// for i, c := range expectedCids {
// fmt.Printf("%d: %s\n", i, c)
// }
go listenErrCh(t, errChan)
// for c := range resultCids {
// fmt.Printf("%s\n", c)
// }
go func() { // listen on outChan so progress can be made
for {
_, ok := <-outChan
if !ok {
// channel closed, safe to exit
return
}
}
}()
objs := make([]interface{}, 0)
for obj := range printChan {
objs = append(objs, obj)
if len(expectedCids) != len(resultCids) {
t.Fatal("unexpected number of blocks imported")
}
testChannelOutput(t, objs, test.TestDirCids[:15])
}
// listenErrCh listens on the error channel until closed and raise any errors
// that show up
func listenErrCh(t *testing.T, errChan <-chan error) {
for {
err, ok := <-errChan
for _, c := range expectedCids {
_, ok := resultCids[c]
if !ok {
// channel closed, safe to exit
return
t.Fatal("unexpected block emitted:", c)
}
}
}
func TestImporter_DoubleStart(t *testing.T) {
f := test.GetTestingDirSerial(t)
p := DefaultParams()
imp, err := NewImporter(f, p)
if err != nil {
t.Fatal(err)
}
}
// testChannelOutput is a utility for shared functionality of output and print
// channel testing
func testChannelOutput(t *testing.T, objs []interface{}, expected []string) {
check := make(map[string]struct{})
for _, obj := range objs {
var cid string
switch obj := obj.(type) {
case *api.AddedOutput:
cid = obj.Hash
case *api.NodeWithMeta:
cid = obj.Cid
}
if _, ok := check[cid]; ok {
t.Fatalf("Duplicate cid %s", cid)
}
check[cid] = struct{}{}
blockHandler := func(ctx context.Context, n *api.NodeWithMeta) (string, error) {
return "", nil
}
if len(check) != len(expected) {
t.Fatalf("Witnessed cids: %v\nExpected cids: %v", check, test.TestDirCids[:15])
}
for cid := range check {
if !contains(expected, cid) {
t.Fatalf("Unexpected cid: %s", cid)
}
}
}
func contains(slice []string, s string) bool {
for _, a := range slice {
if a == s {
return true
}
_, err = imp.Run(context.Background(), blockHandler)
if err != nil {
t.Fatal(err)
}
_, err = imp.Run(context.Background(), blockHandler)
if err == nil {
t.Fatal("expected an error: cannot run importer twice")
}
return false
}

92
adder/local/adder_test.go Normal file
View File

@ -0,0 +1,92 @@
package local
import (
"context"
"mime/multipart"
"sync"
"testing"
"github.com/ipfs/ipfs-cluster/adder"
"github.com/ipfs/ipfs-cluster/api"
"github.com/ipfs/ipfs-cluster/test"
rpc "github.com/hsanjuan/go-libp2p-gorpc"
)
type testRPC struct {
blocks sync.Map
pins sync.Map
}
func (rpcs *testRPC) IPFSBlockPut(ctx context.Context, in api.NodeWithMeta, out *struct{}) error {
rpcs.blocks.Store(in.Cid, in)
return nil
}
func (rpcs *testRPC) Pin(ctx context.Context, in api.PinSerial, out *struct{}) error {
rpcs.pins.Store(in.Cid, in)
return nil
}
func TestFromMultipart(t *testing.T) {
defer test.CleanShardingDir(t)
t.Run("balanced", func(t *testing.T) {
rpcObj := &testRPC{}
server := rpc.NewServer(nil, "mock")
err := server.RegisterName("Cluster", rpcObj)
if err != nil {
t.Fatal(err)
}
client := rpc.NewClientWithServer(nil, "mock", server)
add := New(client)
mr := test.GetShardingDirMultiReader(t)
r := multipart.NewReader(mr, mr.Boundary())
err = add.FromMultipart(context.Background(), r, adder.DefaultParams())
if err != nil {
t.Fatal(err)
}
expected := test.ShardingDirCids[:]
for _, c := range expected {
_, ok := rpcObj.blocks.Load(c)
if !ok {
t.Error("no IPFSBlockPut for block", c)
}
}
_, ok := rpcObj.pins.Load(test.ShardingDirBalancedRootCID)
if !ok {
t.Error("the tree wasn't pinned")
}
})
t.Run("trickle", func(t *testing.T) {
rpcObj := &testRPC{}
server := rpc.NewServer(nil, "mock")
err := server.RegisterName("Cluster", rpcObj)
if err != nil {
t.Fatal(err)
}
client := rpc.NewClientWithServer(nil, "mock", server)
add := New(client)
mr := test.GetShardingDirMultiReader(t)
r := multipart.NewReader(mr, mr.Boundary())
p := adder.DefaultParams()
p.Layout = "trickle"
err = add.FromMultipart(context.Background(), r, p)
if err != nil {
t.Fatal(err)
}
_, ok := rpcObj.pins.Load(test.ShardingDirTrickleRootCID)
if !ok {
t.Error("the tree wasn't pinned")
}
})
}

29
adder/params_test.go Normal file
View File

@ -0,0 +1,29 @@
package adder
import (
"net/url"
"testing"
)
func TestParamsFromQuery(t *testing.T) {
qStr := "layout=balanced&chunker=size-262144&name=test&raw=true&hidden=true&shard=true&repl_min=2&repl_max=4&shard_size=1"
q, err := url.ParseQuery(qStr)
if err != nil {
t.Fatal(err)
}
p, err := ParamsFromQuery(q)
if err != nil {
t.Fatal(err)
}
if p.Layout != "balanced" ||
p.Chunker != "size-262144" ||
p.Name != "test" ||
!p.RawLeaves || !p.Hidden || !p.Shard ||
p.ReplicationFactorMin != 2 ||
p.ReplicationFactorMax != 4 ||
p.ShardSize != 1 {
t.Fatal("did not parse the query correctly")
}
}

View File

@ -45,45 +45,3 @@ func MustDecodeCid(v string) *cid.Cid {
c, _ := cid.Decode(v)
return c
}
// Variables related to adding the testing directory generated by tests
var (
NumTestDirPrints = 15
TestDirBalancedRootCID = "QmX2Erb4SBNfcv8X5MFa4z1jGPfaaYA1snMUhQyYVdDqTA"
TestDirTrickleRootCID = "QmQnpSRdGDhUoGiCXQLMaAWmJKgkZbBeVi1pzsQh1xuTGz"
// Hashes of test files generated in this module through calls to GetTestingDir*
// and ipfs add with default chunking and layout
TestDirCids = [29]string{
"QmdWLMAyMF23KyBfAXbNzy7sDu3zGGu27eGQRkQTPMqfoE",
"QmbiPudqP8264Ccia1iyTebFrtGmG3JCW85YmT5Gds1Wt9",
"QmesCMDCHRETdDYfuGjUaGVZGSE2nGxGETPoBUgwitnsCT",
"Qmbiz4Y6ByNTrzEwE2veqy7S8gUBJNNvNqxAy6bBShpvr4",
"QmNtq6PD9Ef6V1UtikhemHBcupjsvr2sDu7bu2DrBSoHWw",
"QmVorRQhT4DbND8JyhAW7HkNPd7bUzqof8XJKcfGcGmvHF",
"QmanFw3Fn96DkMc9XSuhZdvXWDk37cpLrKA6L54MniGL9Z",
"QmRVFNBFwtUKpE7u3Bbd6Nj1QsnyHgryZSTM86bBuphPAn",
"QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
"QmPZLJ3CZYgxH4K1w5jdbAdxJynXn5TCB4kHy7u8uHC3fy",
"QmQntQGk13CkPgr1b3RAseJFaTpVMqQu8zgh21ox1RnwBf",
"QmbR4x5HwcQLiipfyHazhKYA1Z2yN9burWWdAKJBhoZnK3",
"QmYdHmrwb9Wd8kkjLg4yKr7EPqKNGx5vHuREU5HNc7sxnk",
"QmVMmDqWhdH8d4QWyhkkVHYvQYara6ijgCg3PNpvRhbmHo",
"QmX2Erb4SBNfcv8X5MFa4z1jGPfaaYA1snMUhQyYVdDqTA",
// Hashes that are not printed out (chunks of files)
"QmavW3cdGuSfYMEQiBDfobwVtPEjUnML2Ry1q8w8X3Q8Wj",
"QmfPHRbeerRWgbu5BzxwK7UhmJGqGvZNxuFoMCUFTuhG3H",
"QmaYNfhw7L7KWX7LYpwWt1bh6Gq2p7z1tic35PnDRnqyBf",
"QmWWwH1GKMh6GmFQunjq7CHjr4g4z6Q4xHyDVfuZGX7MyU",
"QmVpHQGMF5PLsvfgj8bGo9q2YyLRPMvfu1uTb3DgREFtUc",
"QmUrdAn4Mx4kNioX9juLgwQotwFfxeo5doUNnLJrQynBEN",
"QmdJ86B7J8mfGq6SjQy8Jz7r5x1cLcXc9M2a7T7NmSMVZx",
"QmS77cTMdyx8P7rP2Gij6azgYPpjp2J34EVYuhB6mfjrQh",
"QmbsBsDspFcqi7xJ4xPxcNYnduzQ5UQDw9y6trQWZGoEHq",
"QmakAXHMeyE6fHHaeqicSKVMM2QyuGbS2g8dgUA7ns8gSY",
"QmTC6vGbH9ABkpXfrMmYkXbxEqH12jEVGpvGzibGZEDVHK",
"QmRLF116yZdLLw7bLvjnHxXVHrjB2snNoJ1itpQxi8NkZP",
"QmZ2iUT3W7jh8QNnpWSiMZ1QYgpommCSQFZiPY5VdoCHyv",
"QmR5mq8smc6zCvo3eRhS47ScgEwKpPw7b1joExysyqgbee",
}
)

View File

@ -2,53 +2,100 @@ package test
import (
"bufio"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
"testing"
"github.com/ipfs/go-ipfs-cmdkit/files"
)
const testDirName = "testingData"
const relRootDir = "src/github.com/ipfs/ipfs-cluster/test"
const shardingDirName = "shardTesting"
// GetTestingDirSerial returns a cmdkits serial file to the testing directory.
// $GOPATH must be set for this to work
func GetTestingDirSerial() (files.File, error) {
rootPath := filepath.Join(os.Getenv("GOPATH"), relRootDir)
testDir := filepath.Join(rootPath, testDirName)
// Variables related to adding the testing directory generated by tests
var (
NumShardingDirPrints = 15
ShardingDirBalancedRootCID = "QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk"
ShardingDirTrickleRootCID = "QmaNXYLz6LbPMrEYcm9Mot1HoHGYbsn7JvzLPk6C9ubAcp"
if _, err := os.Stat(testDir); os.IsNotExist(err) {
err := generateTestDirs(rootPath)
// These hashes should match all the blocks produced when adding
// the files resulting from GetShardingDir*
// They have been obtained by adding the "shardTesting" folder
// to go-ipfs (with wrap=true and default parameters). Then doing
// `refs -r` on the result. It contains the wrapping folder hash.
ShardingDirCids = [29]string{
"QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk",
"QmVMmDqWhdH8d4QWyhkkVHYvQYara6ijgCg3PNpvRhbmHo",
"QmbR4x5HwcQLiipfyHazhKYA1Z2yN9burWWdAKJBhoZnK3",
"QmanFw3Fn96DkMc9XSuhZdvXWDk37cpLrKA6L54MniGL9Z",
"QmdWLMAyMF23KyBfAXbNzy7sDu3zGGu27eGQRkQTPMqfoE",
"QmRVFNBFwtUKpE7u3Bbd6Nj1QsnyHgryZSTM86bBuphPAn",
"QmbiPudqP8264Ccia1iyTebFrtGmG3JCW85YmT5Gds1Wt9",
"QmPZLJ3CZYgxH4K1w5jdbAdxJynXn5TCB4kHy7u8uHC3fy",
"QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
"QmQntQGk13CkPgr1b3RAseJFaTpVMqQu8zgh21ox1RnwBf",
"QmesCMDCHRETdDYfuGjUaGVZGSE2nGxGETPoBUgwitnsCT",
"Qmbiz4Y6ByNTrzEwE2veqy7S8gUBJNNvNqxAy6bBShpvr4",
"QmYdHmrwb9Wd8kkjLg4yKr7EPqKNGx5vHuREU5HNc7sxnk",
"QmNtq6PD9Ef6V1UtikhemHBcupjsvr2sDu7bu2DrBSoHWw",
"QmavW3cdGuSfYMEQiBDfobwVtPEjUnML2Ry1q8w8X3Q8Wj",
"QmfPHRbeerRWgbu5BzxwK7UhmJGqGvZNxuFoMCUFTuhG3H",
"QmaYNfhw7L7KWX7LYpwWt1bh6Gq2p7z1tic35PnDRnqyBf",
"QmWWwH1GKMh6GmFQunjq7CHjr4g4z6Q4xHyDVfuZGX7MyU",
"QmVpHQGMF5PLsvfgj8bGo9q2YyLRPMvfu1uTb3DgREFtUc",
"QmUrdAn4Mx4kNioX9juLgwQotwFfxeo5doUNnLJrQynBEN",
"QmdJ86B7J8mfGq6SjQy8Jz7r5x1cLcXc9M2a7T7NmSMVZx",
"QmS77cTMdyx8P7rP2Gij6azgYPpjp2J34EVYuhB6mfjrQh",
"QmbsBsDspFcqi7xJ4xPxcNYnduzQ5UQDw9y6trQWZGoEHq",
"QmakAXHMeyE6fHHaeqicSKVMM2QyuGbS2g8dgUA7ns8gSY",
"QmTC6vGbH9ABkpXfrMmYkXbxEqH12jEVGpvGzibGZEDVHK",
"QmRLF116yZdLLw7bLvjnHxXVHrjB2snNoJ1itpQxi8NkZP",
"QmVorRQhT4DbND8JyhAW7HkNPd7bUzqof8XJKcfGcGmvHF",
"QmZ2iUT3W7jh8QNnpWSiMZ1QYgpommCSQFZiPY5VdoCHyv",
"QmR5mq8smc6zCvo3eRhS47ScgEwKpPw7b1joExysyqgbee",
}
)
// CleanShardingDir deletes the folders generated with GetShardingDir() and
// GetShardingMultiReader.
func CleanShardingDir(t *testing.T) {
os.RemoveAll(shardingDirName)
}
// GetShardingDirSerial returns a cmdkits serial file to the testing directory.
func GetShardingDirSerial(t *testing.T) files.File {
if _, err := os.Stat(shardingDirName); os.IsNotExist(err) {
err := generateShardingDirs(shardingDirName)
if err != nil {
return nil, err
t.Fatal(err)
}
}
stat, err := os.Lstat(testDir)
stat, err := os.Lstat(shardingDirName)
if err != nil {
return nil, err
t.Fatal(err)
}
if !stat.IsDir() {
return nil, fmt.Errorf("testDir should be seen as directory")
t.Fatal("testDir should be seen as directory")
}
return files.NewSerialFile(testDirName, testDir, false, stat)
}
// GetTestingDirMultiReader returns a cmdkits multifilereader to the testing
// directory. $GOPATH must be set for this to work
func GetTestingDirMultiReader() (*files.MultiFileReader, error) {
file, err := GetTestingDirSerial()
f, err := files.NewSerialFile(shardingDirName, shardingDirName, false, stat)
if err != nil {
return nil, err
t.Fatal(err)
}
sliceFile := files.NewSliceFile("", "", []files.File{file})
return files.NewMultiFileReader(sliceFile, true), nil
return f
}
// generateTestDirs creates a testing directory structure on demand for testing
// GetShardingDirMultiReader returns a cmdkits multifilereader to the testing
// directory.
func GetShardingDirMultiReader(t *testing.T) *files.MultiFileReader {
file := GetShardingDirSerial(t)
sliceFile := files.NewSliceFile("", "", []files.File{file})
mfr := files.NewMultiFileReader(sliceFile, true)
return mfr
}
// generateShardingDirs creates a testing directory structure on demand for testing
// leveraging random but deterministic strings. Files are the same every run.
// Directory structure:
// - testingData
@ -69,20 +116,19 @@ func GetTestingDirMultiReader() (*files.MultiFileReader, error) {
// Take special care when modifying this function. File data depends on order
// and each file size. If this changes then hashes stored in test/cids.go
// recording the ipfs import hash tree must be updated manually.
func generateTestDirs(path string) error {
func generateShardingDirs(path string) error {
// Prepare randomness source for writing files
src := rand.NewSource(int64(1))
ra := rand.New(src)
// Make top level directory
rootPath := filepath.Join(path, testDirName)
err := os.Mkdir(rootPath, os.ModePerm)
err := os.Mkdir(path, os.ModePerm)
if err != nil {
return err
}
// Make subdir A
aPath := filepath.Join(rootPath, "A")
aPath := filepath.Join(path, "A")
err = os.Mkdir(aPath, os.ModePerm)
if err != nil {
return err
@ -171,7 +217,7 @@ func generateTestDirs(path string) error {
}
// Make subdir B
bPath := filepath.Join(rootPath, "B")
bPath := filepath.Join(path, "B")
err = os.Mkdir(bPath, os.ModePerm)
if err != nil {
return err