wip
License: MIT Signed-off-by: Hector Sanjuan <code@hector.link>
This commit is contained in:
parent
a96241941e
commit
9d89bda2a6
1
.gitignore
vendored
1
.gitignore
vendored
|
@ -9,6 +9,7 @@ sharness/trash*
|
|||
|
||||
raftFolderFromTest*
|
||||
peerstore
|
||||
shardTesting
|
||||
|
||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
|
|
|
@ -100,24 +100,24 @@ func (dag *adderDAGService) RemoveMany(ctx context.Context, keys []*cid.Cid) err
|
|||
return nil
|
||||
}
|
||||
|
||||
// printDAGService will "add" a node by printing it. printDAGService cannot Get nodes
|
||||
// that have already been seen and calls to Remove are noops. Nodes are
|
||||
// recorded after being added so that they will only be printed once.
|
||||
type printDAGService struct {
|
||||
ads ipld.DAGService
|
||||
}
|
||||
// // printDAGService will "add" a node by printing it. printDAGService cannot Get nodes
|
||||
// // that have already been seen and calls to Remove are noops. Nodes are
|
||||
// // recorded after being added so that they will only be printed once.
|
||||
// type printDAGService struct {
|
||||
// ads ipld.DAGService
|
||||
// }
|
||||
|
||||
func newPDagService() *printDAGService {
|
||||
ch := make(chan *api.NodeWithMeta)
|
||||
ads := newAdderDAGService(ch)
|
||||
// func newPDagService() *printDAGService {
|
||||
// ch := make(chan *api.NodeWithMeta)
|
||||
// ads := newAdderDAGService(ch)
|
||||
|
||||
go func() {
|
||||
for n := range ch {
|
||||
fmt.Printf(n.Cid, " | ", n.Size)
|
||||
}
|
||||
}()
|
||||
// go func() {
|
||||
// for n := range ch {
|
||||
// fmt.Printf(n.Cid, " | ", n.Size)
|
||||
// }
|
||||
// }()
|
||||
|
||||
return &printDAGService{
|
||||
ads: ads,
|
||||
}
|
||||
}
|
||||
// return &printDAGService{
|
||||
// ads: ads,
|
||||
// }
|
||||
// }
|
||||
|
|
|
@ -108,26 +108,26 @@ func (imp *Importer) Go(ctx context.Context) error {
|
|||
case <-ctx.Done():
|
||||
imp.errors <- ctx.Err()
|
||||
return
|
||||
}
|
||||
|
||||
f, err := imp.files.NextFile()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break // time to finalize
|
||||
default:
|
||||
f, err := imp.files.NextFile()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
goto FINALIZE // time to finalize
|
||||
}
|
||||
imp.errors <- err
|
||||
return
|
||||
}
|
||||
imp.errors <- err
|
||||
return
|
||||
}
|
||||
|
||||
if err := ipfsAdder.AddFile(f); err != nil {
|
||||
imp.errors <- err
|
||||
return
|
||||
if err := ipfsAdder.AddFile(f); err != nil {
|
||||
imp.errors <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FINALIZE:
|
||||
_, err := ipfsAdder.Finalize()
|
||||
if err != nil {
|
||||
if !isNotFound(err) {
|
||||
if isNotFound(err) {
|
||||
fmt.Println("fixme importer.go", err)
|
||||
} else {
|
||||
imp.errors <- err
|
||||
|
@ -144,6 +144,7 @@ func (imp *Importer) Run(ctx context.Context, blockF BlockHandler) (string, erro
|
|||
|
||||
errors := imp.Errors()
|
||||
blocks := imp.Blocks()
|
||||
output := imp.Output()
|
||||
|
||||
err := imp.Go(ctx)
|
||||
if err != nil {
|
||||
|
@ -156,19 +157,22 @@ func (imp *Importer) Run(ctx context.Context, blockF BlockHandler) (string, erro
|
|||
return retVal, ctx.Err()
|
||||
case err, ok := <-errors:
|
||||
if ok {
|
||||
fmt.Println(err)
|
||||
return retVal, err
|
||||
}
|
||||
case node, ok := <-blocks:
|
||||
if !ok {
|
||||
break // finished importing
|
||||
goto BREAK // finished importing
|
||||
}
|
||||
retVal, err := blockF(ctx, node)
|
||||
retVal, err = blockF(ctx, node)
|
||||
if err != nil {
|
||||
return retVal, err
|
||||
}
|
||||
case <-output:
|
||||
// TODO
|
||||
}
|
||||
}
|
||||
|
||||
BREAK:
|
||||
// grab any last errors from errors if necessary
|
||||
// (this waits for errors to be closed)
|
||||
err = <-errors
|
||||
|
|
|
@ -8,107 +8,69 @@ import (
|
|||
"github.com/ipfs/ipfs-cluster/test"
|
||||
)
|
||||
|
||||
// import and receive all blocks
|
||||
func TestToChannelOutput(t *testing.T) {
|
||||
file, err := test.GetTestingDirSerial()
|
||||
func TestImporter(t *testing.T) {
|
||||
f := test.GetTestingDirSerial(t)
|
||||
p := DefaultParams()
|
||||
|
||||
imp, err := NewImporter(f, p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
printChan, outChan, errChan := ToChannel(context.Background(), file,
|
||||
false, false, false, "")
|
||||
expectedCids := test.TestDirCids[:]
|
||||
resultCids := make(map[string]struct{})
|
||||
|
||||
go func() { // listen on printChan so progress can be made
|
||||
for {
|
||||
_, ok := <-printChan
|
||||
if !ok {
|
||||
// channel closed, safe to exit
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go listenErrCh(t, errChan)
|
||||
|
||||
objs := make([]interface{}, 0)
|
||||
for obj := range outChan {
|
||||
objs = append(objs, obj)
|
||||
blockHandler := func(ctx context.Context, n *api.NodeWithMeta) (string, error) {
|
||||
resultCids[n.Cid] = struct{}{}
|
||||
return n.Cid, nil
|
||||
}
|
||||
testChannelOutput(t, objs, test.TestDirCids[:])
|
||||
}
|
||||
|
||||
func TestToChannelPrint(t *testing.T) {
|
||||
file, err := test.GetTestingDirSerial()
|
||||
_, err = imp.Run(context.Background(), blockHandler)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
printChan, outChan, errChan := ToChannel(context.Background(), file,
|
||||
false, false, false, "")
|
||||
// for i, c := range expectedCids {
|
||||
// fmt.Printf("%d: %s\n", i, c)
|
||||
// }
|
||||
|
||||
go listenErrCh(t, errChan)
|
||||
// for c := range resultCids {
|
||||
// fmt.Printf("%s\n", c)
|
||||
// }
|
||||
|
||||
go func() { // listen on outChan so progress can be made
|
||||
for {
|
||||
_, ok := <-outChan
|
||||
if !ok {
|
||||
// channel closed, safe to exit
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
objs := make([]interface{}, 0)
|
||||
for obj := range printChan {
|
||||
objs = append(objs, obj)
|
||||
if len(expectedCids) != len(resultCids) {
|
||||
t.Fatal("unexpected number of blocks imported")
|
||||
}
|
||||
testChannelOutput(t, objs, test.TestDirCids[:15])
|
||||
}
|
||||
|
||||
// listenErrCh listens on the error channel until closed and raise any errors
|
||||
// that show up
|
||||
func listenErrCh(t *testing.T, errChan <-chan error) {
|
||||
for {
|
||||
err, ok := <-errChan
|
||||
for _, c := range expectedCids {
|
||||
_, ok := resultCids[c]
|
||||
if !ok {
|
||||
// channel closed, safe to exit
|
||||
return
|
||||
t.Fatal("unexpected block emitted:", c)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestImporter_DoubleStart(t *testing.T) {
|
||||
f := test.GetTestingDirSerial(t)
|
||||
p := DefaultParams()
|
||||
|
||||
imp, err := NewImporter(f, p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// testChannelOutput is a utility for shared functionality of output and print
|
||||
// channel testing
|
||||
func testChannelOutput(t *testing.T, objs []interface{}, expected []string) {
|
||||
check := make(map[string]struct{})
|
||||
for _, obj := range objs {
|
||||
var cid string
|
||||
switch obj := obj.(type) {
|
||||
case *api.AddedOutput:
|
||||
cid = obj.Hash
|
||||
case *api.NodeWithMeta:
|
||||
cid = obj.Cid
|
||||
}
|
||||
if _, ok := check[cid]; ok {
|
||||
t.Fatalf("Duplicate cid %s", cid)
|
||||
}
|
||||
check[cid] = struct{}{}
|
||||
blockHandler := func(ctx context.Context, n *api.NodeWithMeta) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
if len(check) != len(expected) {
|
||||
t.Fatalf("Witnessed cids: %v\nExpected cids: %v", check, test.TestDirCids[:15])
|
||||
}
|
||||
for cid := range check {
|
||||
if !contains(expected, cid) {
|
||||
t.Fatalf("Unexpected cid: %s", cid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func contains(slice []string, s string) bool {
|
||||
for _, a := range slice {
|
||||
if a == s {
|
||||
return true
|
||||
}
|
||||
_, err = imp.Run(context.Background(), blockHandler)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = imp.Run(context.Background(), blockHandler)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error: cannot run importer twice")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
92
adder/local/adder_test.go
Normal file
92
adder/local/adder_test.go
Normal file
|
@ -0,0 +1,92 @@
|
|||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"mime/multipart"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/ipfs-cluster/adder"
|
||||
"github.com/ipfs/ipfs-cluster/api"
|
||||
"github.com/ipfs/ipfs-cluster/test"
|
||||
|
||||
rpc "github.com/hsanjuan/go-libp2p-gorpc"
|
||||
)
|
||||
|
||||
type testRPC struct {
|
||||
blocks sync.Map
|
||||
pins sync.Map
|
||||
}
|
||||
|
||||
func (rpcs *testRPC) IPFSBlockPut(ctx context.Context, in api.NodeWithMeta, out *struct{}) error {
|
||||
rpcs.blocks.Store(in.Cid, in)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rpcs *testRPC) Pin(ctx context.Context, in api.PinSerial, out *struct{}) error {
|
||||
rpcs.pins.Store(in.Cid, in)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFromMultipart(t *testing.T) {
|
||||
defer test.CleanShardingDir(t)
|
||||
|
||||
t.Run("balanced", func(t *testing.T) {
|
||||
rpcObj := &testRPC{}
|
||||
server := rpc.NewServer(nil, "mock")
|
||||
err := server.RegisterName("Cluster", rpcObj)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := rpc.NewClientWithServer(nil, "mock", server)
|
||||
|
||||
add := New(client)
|
||||
|
||||
mr := test.GetShardingDirMultiReader(t)
|
||||
r := multipart.NewReader(mr, mr.Boundary())
|
||||
err = add.FromMultipart(context.Background(), r, adder.DefaultParams())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
expected := test.ShardingDirCids[:]
|
||||
for _, c := range expected {
|
||||
_, ok := rpcObj.blocks.Load(c)
|
||||
if !ok {
|
||||
t.Error("no IPFSBlockPut for block", c)
|
||||
}
|
||||
}
|
||||
|
||||
_, ok := rpcObj.pins.Load(test.ShardingDirBalancedRootCID)
|
||||
if !ok {
|
||||
t.Error("the tree wasn't pinned")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("trickle", func(t *testing.T) {
|
||||
rpcObj := &testRPC{}
|
||||
server := rpc.NewServer(nil, "mock")
|
||||
err := server.RegisterName("Cluster", rpcObj)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := rpc.NewClientWithServer(nil, "mock", server)
|
||||
|
||||
add := New(client)
|
||||
|
||||
mr := test.GetShardingDirMultiReader(t)
|
||||
r := multipart.NewReader(mr, mr.Boundary())
|
||||
p := adder.DefaultParams()
|
||||
p.Layout = "trickle"
|
||||
|
||||
err = add.FromMultipart(context.Background(), r, p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, ok := rpcObj.pins.Load(test.ShardingDirTrickleRootCID)
|
||||
if !ok {
|
||||
t.Error("the tree wasn't pinned")
|
||||
}
|
||||
})
|
||||
}
|
29
adder/params_test.go
Normal file
29
adder/params_test.go
Normal file
|
@ -0,0 +1,29 @@
|
|||
package adder
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParamsFromQuery(t *testing.T) {
|
||||
qStr := "layout=balanced&chunker=size-262144&name=test&raw=true&hidden=true&shard=true&repl_min=2&repl_max=4&shard_size=1"
|
||||
|
||||
q, err := url.ParseQuery(qStr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := ParamsFromQuery(q)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Layout != "balanced" ||
|
||||
p.Chunker != "size-262144" ||
|
||||
p.Name != "test" ||
|
||||
!p.RawLeaves || !p.Hidden || !p.Shard ||
|
||||
p.ReplicationFactorMin != 2 ||
|
||||
p.ReplicationFactorMax != 4 ||
|
||||
p.ShardSize != 1 {
|
||||
t.Fatal("did not parse the query correctly")
|
||||
}
|
||||
}
|
42
test/cids.go
42
test/cids.go
|
@ -45,45 +45,3 @@ func MustDecodeCid(v string) *cid.Cid {
|
|||
c, _ := cid.Decode(v)
|
||||
return c
|
||||
}
|
||||
|
||||
// Variables related to adding the testing directory generated by tests
|
||||
var (
|
||||
NumTestDirPrints = 15
|
||||
TestDirBalancedRootCID = "QmX2Erb4SBNfcv8X5MFa4z1jGPfaaYA1snMUhQyYVdDqTA"
|
||||
TestDirTrickleRootCID = "QmQnpSRdGDhUoGiCXQLMaAWmJKgkZbBeVi1pzsQh1xuTGz"
|
||||
|
||||
// Hashes of test files generated in this module through calls to GetTestingDir*
|
||||
// and ipfs add with default chunking and layout
|
||||
TestDirCids = [29]string{
|
||||
"QmdWLMAyMF23KyBfAXbNzy7sDu3zGGu27eGQRkQTPMqfoE",
|
||||
"QmbiPudqP8264Ccia1iyTebFrtGmG3JCW85YmT5Gds1Wt9",
|
||||
"QmesCMDCHRETdDYfuGjUaGVZGSE2nGxGETPoBUgwitnsCT",
|
||||
"Qmbiz4Y6ByNTrzEwE2veqy7S8gUBJNNvNqxAy6bBShpvr4",
|
||||
"QmNtq6PD9Ef6V1UtikhemHBcupjsvr2sDu7bu2DrBSoHWw",
|
||||
"QmVorRQhT4DbND8JyhAW7HkNPd7bUzqof8XJKcfGcGmvHF",
|
||||
"QmanFw3Fn96DkMc9XSuhZdvXWDk37cpLrKA6L54MniGL9Z",
|
||||
"QmRVFNBFwtUKpE7u3Bbd6Nj1QsnyHgryZSTM86bBuphPAn",
|
||||
"QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
|
||||
"QmPZLJ3CZYgxH4K1w5jdbAdxJynXn5TCB4kHy7u8uHC3fy",
|
||||
"QmQntQGk13CkPgr1b3RAseJFaTpVMqQu8zgh21ox1RnwBf",
|
||||
"QmbR4x5HwcQLiipfyHazhKYA1Z2yN9burWWdAKJBhoZnK3",
|
||||
"QmYdHmrwb9Wd8kkjLg4yKr7EPqKNGx5vHuREU5HNc7sxnk",
|
||||
"QmVMmDqWhdH8d4QWyhkkVHYvQYara6ijgCg3PNpvRhbmHo",
|
||||
"QmX2Erb4SBNfcv8X5MFa4z1jGPfaaYA1snMUhQyYVdDqTA",
|
||||
// Hashes that are not printed out (chunks of files)
|
||||
"QmavW3cdGuSfYMEQiBDfobwVtPEjUnML2Ry1q8w8X3Q8Wj",
|
||||
"QmfPHRbeerRWgbu5BzxwK7UhmJGqGvZNxuFoMCUFTuhG3H",
|
||||
"QmaYNfhw7L7KWX7LYpwWt1bh6Gq2p7z1tic35PnDRnqyBf",
|
||||
"QmWWwH1GKMh6GmFQunjq7CHjr4g4z6Q4xHyDVfuZGX7MyU",
|
||||
"QmVpHQGMF5PLsvfgj8bGo9q2YyLRPMvfu1uTb3DgREFtUc",
|
||||
"QmUrdAn4Mx4kNioX9juLgwQotwFfxeo5doUNnLJrQynBEN",
|
||||
"QmdJ86B7J8mfGq6SjQy8Jz7r5x1cLcXc9M2a7T7NmSMVZx",
|
||||
"QmS77cTMdyx8P7rP2Gij6azgYPpjp2J34EVYuhB6mfjrQh",
|
||||
"QmbsBsDspFcqi7xJ4xPxcNYnduzQ5UQDw9y6trQWZGoEHq",
|
||||
"QmakAXHMeyE6fHHaeqicSKVMM2QyuGbS2g8dgUA7ns8gSY",
|
||||
"QmTC6vGbH9ABkpXfrMmYkXbxEqH12jEVGpvGzibGZEDVHK",
|
||||
"QmRLF116yZdLLw7bLvjnHxXVHrjB2snNoJ1itpQxi8NkZP",
|
||||
"QmZ2iUT3W7jh8QNnpWSiMZ1QYgpommCSQFZiPY5VdoCHyv",
|
||||
"QmR5mq8smc6zCvo3eRhS47ScgEwKpPw7b1joExysyqgbee",
|
||||
}
|
||||
)
|
||||
|
|
|
@ -2,53 +2,100 @@ package test
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ipfs/go-ipfs-cmdkit/files"
|
||||
)
|
||||
|
||||
const testDirName = "testingData"
|
||||
const relRootDir = "src/github.com/ipfs/ipfs-cluster/test"
|
||||
const shardingDirName = "shardTesting"
|
||||
|
||||
// GetTestingDirSerial returns a cmdkits serial file to the testing directory.
|
||||
// $GOPATH must be set for this to work
|
||||
func GetTestingDirSerial() (files.File, error) {
|
||||
rootPath := filepath.Join(os.Getenv("GOPATH"), relRootDir)
|
||||
testDir := filepath.Join(rootPath, testDirName)
|
||||
// Variables related to adding the testing directory generated by tests
|
||||
var (
|
||||
NumShardingDirPrints = 15
|
||||
ShardingDirBalancedRootCID = "QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk"
|
||||
ShardingDirTrickleRootCID = "QmaNXYLz6LbPMrEYcm9Mot1HoHGYbsn7JvzLPk6C9ubAcp"
|
||||
|
||||
if _, err := os.Stat(testDir); os.IsNotExist(err) {
|
||||
err := generateTestDirs(rootPath)
|
||||
// These hashes should match all the blocks produced when adding
|
||||
// the files resulting from GetShardingDir*
|
||||
// They have been obtained by adding the "shardTesting" folder
|
||||
// to go-ipfs (with wrap=true and default parameters). Then doing
|
||||
// `refs -r` on the result. It contains the wrapping folder hash.
|
||||
ShardingDirCids = [29]string{
|
||||
"QmUtFSRGDUQ1mHAsX8udixU5rn8e34Lqm5pJBoUpyXPumk",
|
||||
"QmVMmDqWhdH8d4QWyhkkVHYvQYara6ijgCg3PNpvRhbmHo",
|
||||
"QmbR4x5HwcQLiipfyHazhKYA1Z2yN9burWWdAKJBhoZnK3",
|
||||
"QmanFw3Fn96DkMc9XSuhZdvXWDk37cpLrKA6L54MniGL9Z",
|
||||
"QmdWLMAyMF23KyBfAXbNzy7sDu3zGGu27eGQRkQTPMqfoE",
|
||||
"QmRVFNBFwtUKpE7u3Bbd6Nj1QsnyHgryZSTM86bBuphPAn",
|
||||
"QmbiPudqP8264Ccia1iyTebFrtGmG3JCW85YmT5Gds1Wt9",
|
||||
"QmPZLJ3CZYgxH4K1w5jdbAdxJynXn5TCB4kHy7u8uHC3fy",
|
||||
"QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn",
|
||||
"QmQntQGk13CkPgr1b3RAseJFaTpVMqQu8zgh21ox1RnwBf",
|
||||
"QmesCMDCHRETdDYfuGjUaGVZGSE2nGxGETPoBUgwitnsCT",
|
||||
"Qmbiz4Y6ByNTrzEwE2veqy7S8gUBJNNvNqxAy6bBShpvr4",
|
||||
"QmYdHmrwb9Wd8kkjLg4yKr7EPqKNGx5vHuREU5HNc7sxnk",
|
||||
"QmNtq6PD9Ef6V1UtikhemHBcupjsvr2sDu7bu2DrBSoHWw",
|
||||
"QmavW3cdGuSfYMEQiBDfobwVtPEjUnML2Ry1q8w8X3Q8Wj",
|
||||
"QmfPHRbeerRWgbu5BzxwK7UhmJGqGvZNxuFoMCUFTuhG3H",
|
||||
"QmaYNfhw7L7KWX7LYpwWt1bh6Gq2p7z1tic35PnDRnqyBf",
|
||||
"QmWWwH1GKMh6GmFQunjq7CHjr4g4z6Q4xHyDVfuZGX7MyU",
|
||||
"QmVpHQGMF5PLsvfgj8bGo9q2YyLRPMvfu1uTb3DgREFtUc",
|
||||
"QmUrdAn4Mx4kNioX9juLgwQotwFfxeo5doUNnLJrQynBEN",
|
||||
"QmdJ86B7J8mfGq6SjQy8Jz7r5x1cLcXc9M2a7T7NmSMVZx",
|
||||
"QmS77cTMdyx8P7rP2Gij6azgYPpjp2J34EVYuhB6mfjrQh",
|
||||
"QmbsBsDspFcqi7xJ4xPxcNYnduzQ5UQDw9y6trQWZGoEHq",
|
||||
"QmakAXHMeyE6fHHaeqicSKVMM2QyuGbS2g8dgUA7ns8gSY",
|
||||
"QmTC6vGbH9ABkpXfrMmYkXbxEqH12jEVGpvGzibGZEDVHK",
|
||||
"QmRLF116yZdLLw7bLvjnHxXVHrjB2snNoJ1itpQxi8NkZP",
|
||||
"QmVorRQhT4DbND8JyhAW7HkNPd7bUzqof8XJKcfGcGmvHF",
|
||||
"QmZ2iUT3W7jh8QNnpWSiMZ1QYgpommCSQFZiPY5VdoCHyv",
|
||||
"QmR5mq8smc6zCvo3eRhS47ScgEwKpPw7b1joExysyqgbee",
|
||||
}
|
||||
)
|
||||
|
||||
// CleanShardingDir deletes the folders generated with GetShardingDir() and
|
||||
// GetShardingMultiReader.
|
||||
func CleanShardingDir(t *testing.T) {
|
||||
os.RemoveAll(shardingDirName)
|
||||
}
|
||||
|
||||
// GetShardingDirSerial returns a cmdkits serial file to the testing directory.
|
||||
func GetShardingDirSerial(t *testing.T) files.File {
|
||||
if _, err := os.Stat(shardingDirName); os.IsNotExist(err) {
|
||||
err := generateShardingDirs(shardingDirName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
stat, err := os.Lstat(testDir)
|
||||
stat, err := os.Lstat(shardingDirName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !stat.IsDir() {
|
||||
return nil, fmt.Errorf("testDir should be seen as directory")
|
||||
t.Fatal("testDir should be seen as directory")
|
||||
}
|
||||
return files.NewSerialFile(testDirName, testDir, false, stat)
|
||||
}
|
||||
|
||||
// GetTestingDirMultiReader returns a cmdkits multifilereader to the testing
|
||||
// directory. $GOPATH must be set for this to work
|
||||
func GetTestingDirMultiReader() (*files.MultiFileReader, error) {
|
||||
file, err := GetTestingDirSerial()
|
||||
f, err := files.NewSerialFile(shardingDirName, shardingDirName, false, stat)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
t.Fatal(err)
|
||||
}
|
||||
sliceFile := files.NewSliceFile("", "", []files.File{file})
|
||||
return files.NewMultiFileReader(sliceFile, true), nil
|
||||
return f
|
||||
}
|
||||
|
||||
// generateTestDirs creates a testing directory structure on demand for testing
|
||||
// GetShardingDirMultiReader returns a cmdkits multifilereader to the testing
|
||||
// directory.
|
||||
func GetShardingDirMultiReader(t *testing.T) *files.MultiFileReader {
|
||||
file := GetShardingDirSerial(t)
|
||||
sliceFile := files.NewSliceFile("", "", []files.File{file})
|
||||
mfr := files.NewMultiFileReader(sliceFile, true)
|
||||
return mfr
|
||||
}
|
||||
|
||||
// generateShardingDirs creates a testing directory structure on demand for testing
|
||||
// leveraging random but deterministic strings. Files are the same every run.
|
||||
// Directory structure:
|
||||
// - testingData
|
||||
|
@ -69,20 +116,19 @@ func GetTestingDirMultiReader() (*files.MultiFileReader, error) {
|
|||
// Take special care when modifying this function. File data depends on order
|
||||
// and each file size. If this changes then hashes stored in test/cids.go
|
||||
// recording the ipfs import hash tree must be updated manually.
|
||||
func generateTestDirs(path string) error {
|
||||
func generateShardingDirs(path string) error {
|
||||
// Prepare randomness source for writing files
|
||||
src := rand.NewSource(int64(1))
|
||||
ra := rand.New(src)
|
||||
|
||||
// Make top level directory
|
||||
rootPath := filepath.Join(path, testDirName)
|
||||
err := os.Mkdir(rootPath, os.ModePerm)
|
||||
err := os.Mkdir(path, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Make subdir A
|
||||
aPath := filepath.Join(rootPath, "A")
|
||||
aPath := filepath.Join(path, "A")
|
||||
err = os.Mkdir(aPath, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -171,7 +217,7 @@ func generateTestDirs(path string) error {
|
|||
}
|
||||
|
||||
// Make subdir B
|
||||
bPath := filepath.Join(rootPath, "B")
|
||||
bPath := filepath.Join(path, "B")
|
||||
err = os.Mkdir(bPath, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
Loading…
Reference in New Issue
Block a user