5fb2b6ae95
Squashed commit of the following: commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e Merge:d125f69
aaada42 Author: Hector Sanjuan <hsanjuan@users.noreply.github.com> Date: Wed Feb 20 11:02:00 2019 +0000 Merge pull request #634 from ipfs/issue_450 Support PinPath, UnpinPath (resolve before pinning) commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Feb 19 22:16:25 2019 +0530 formatResponse accepts api.Pin and not api.PinSerial commit b5da4bea045865814cc422da71827b44ddd44b90 Merge: ba59036cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Feb 19 21:36:46 2019 +0530 Merge branch 'master' into issue_450 commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f Merge: f002914 d59880c Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Mon Feb 18 08:41:11 2019 +0530 Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450 commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Mon Feb 18 08:31:39 2019 +0530 PinPath: more improvements Added tracing for new methods commit d59880c338eaa8214fe06b4f930a540793d78407 Merge: 0ca4c7cb4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com> Date: Wed Feb 13 15:22:49 2019 +0000 Merge branch 'master' into issue_450 commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030 Merge: d35017aecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Feb 12 13:10:13 2019 +0530 Merge branch 'master' into issue_450 commit d35017a8de91ca9fc9a9a047c48c75134cee9f98 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Feb 12 13:07:25 2019 +0530 PinPath: more improvements - Worth having `PinOptions` as a separate field in the struct and constructing the query in the test with ToQuery() - sharness: "intialization" line can be placed outside the tests at the top commit 68e3b90417ffbad89d41a70ac81d85f9037f8848 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Sun Feb 10 21:43:50 2019 +0530 Using if-continue pattern instead of if-else License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 3c29799f3b85be328b27508332ab92049d8b82f3 Merge: 956790b4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Thu Feb 7 10:25:52 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 956790b381db9858e4194f983e898b07dc51ba66 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Feb 6 21:11:20 2019 +0530 Removing resolved path License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 7191cc46cedfbec116a9746937e28881b50ca044 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Feb 6 16:45:07 2019 +0530 Fix go vet License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Feb 6 16:07:03 2019 +0530 Fixed linting error License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 23c57eb467755a1f21387a1615a7f34e97348053 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Feb 6 09:20:41 2019 +0530 Fixed tests License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4 Merge: 17e555e5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Feb 6 00:07:10 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 17e555e4a7c574413df90aac70c5cc29cab98f54 Author: Hector Sanjuan <code@hector.link> Date: Tue Feb 5 16:58:50 2019 +0000 PinPath: address some feedback + improvements * Changed client's Pin() API and PinPath to be consistent * Added helper methods to turn PinPath to query and back * Make code and tests build * Use TestCidResolved everywhere * Fix cluster.PinPath arguments * Fix formatting of responses with --no-status * Make tests readable and call Fatal when needed * Use a pathTestCases variable commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Feb 5 18:34:26 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) Addressed review comments as in https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit a8b4f181d2d7afed32ee41331dfaab19fd66a173 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Jan 29 22:41:27 2019 +0530 Fixing tests License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit e39b95ca19e4d75506f4f492678245ef13936a44 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Jan 29 14:52:53 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - PinPath and UnpinPath should return api.Pin - PinPath should accept pin options - Removing duplicate logic for Resolve from cluster - And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit d146075126320896665ba58d337a13789f68ea86 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Jan 23 17:08:41 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) PinPath(in both rest and rpc) should return a serializable struct in the form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid" to marshal and unmarshal) License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 1f4869568a8adb450275257154ea3a26d03a30f3 Merge: 7acfd28a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Jan 23 07:18:56 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Jan 22 18:14:32 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - RPC must always use serializable structs - In command, just use pin with path as cid is also a valid path - Addressing many other small review comments as in https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Jan 16 09:36:42 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) Extra logic for path checking should go into resolve so that it can be properly reused Added sharness tests License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 9116bda3534e77bb391d873051bb520a1b01a326 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Jan 16 08:08:07 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) error strings should not be capitalized Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit ca7e61861374f456300a85ddc0374e594f74f963 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Jan 15 23:40:25 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) Tests Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 522fbcd899f01c01680375561a32a87464157c0a Merge: f1a56abf7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Jan 15 10:40:54 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit f1a56ab925fb74c0c44273a4524afa4843cf757f Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Mon Jan 14 20:58:17 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - IPFS Connector should act as a pure IPFS client, any extra logic should go to cluster.go - Use cid.Undef, instead of cid.Cid{} Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Thu Jan 10 08:57:17 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - Separate handlers, methods and rpc apis for PinPath and UnpinPath from Pin and Unpin - Support ipld paths as well Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c Merge: 91ceb47 21170c4 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Jan 9 19:38:35 2019 +0530 Merge branch 'issue_450_old' into HEAD License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 91ceb4796259ca7ef2974ec43e6a278a12796b13 Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Wed Jan 9 19:36:41 2019 +0530 Revert "WIP: Figure out why test does not impleme" This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75. License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> commit 28a3a3f25dce6f296c8cbef86221644c099a7e75 Author: cd10012 <ced361@nyu.edu> Date: Tue Jul 24 23:23:10 2018 -0400 WIP: Figure out why test does not implement IPFSConnector interface... License: MIT Signed-off-by: cd10012 <ced361@nyu.edu> commit 21170c48e77e69583db64544b08120a9baf40d8d Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Date: Tue Jan 8 10:37:59 2019 +0530 Support PinPath, UnpinPath (resolve before pinning) This commit adds API support for pinning using path `POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>` will resolve the path into a cid and perform perform pinning or unpinning Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com> Co-authored-by: Hector Sanjuan <hector@protocol.ai> License: MIT Signed-off-by: Hector Sanjuan <hector@protocol.ai>
888 lines
21 KiB
Go
888 lines
21 KiB
Go
package ipfscluster
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"mime/multipart"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ipfs/ipfs-cluster/adder/sharding"
|
|
"github.com/ipfs/ipfs-cluster/allocator/ascendalloc"
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
"github.com/ipfs/ipfs-cluster/consensus/raft"
|
|
"github.com/ipfs/ipfs-cluster/informer/numpin"
|
|
"github.com/ipfs/ipfs-cluster/state"
|
|
"github.com/ipfs/ipfs-cluster/state/mapstate"
|
|
"github.com/ipfs/ipfs-cluster/test"
|
|
"github.com/ipfs/ipfs-cluster/version"
|
|
|
|
gopath "github.com/ipfs/go-path"
|
|
|
|
cid "github.com/ipfs/go-cid"
|
|
rpc "github.com/libp2p/go-libp2p-gorpc"
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
|
)
|
|
|
|
type mockComponent struct {
|
|
rpcClient *rpc.Client
|
|
}
|
|
|
|
func (c *mockComponent) Shutdown(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func (c *mockComponent) SetClient(client *rpc.Client) {
|
|
c.rpcClient = client
|
|
return
|
|
}
|
|
|
|
type mockAPI struct {
|
|
mockComponent
|
|
}
|
|
|
|
type mockProxy struct {
|
|
mockComponent
|
|
}
|
|
|
|
type mockConnector struct {
|
|
mockComponent
|
|
|
|
pins sync.Map
|
|
blocks sync.Map
|
|
}
|
|
|
|
func (ipfs *mockConnector) ID(ctx context.Context) (api.IPFSID, error) {
|
|
return api.IPFSID{
|
|
ID: test.TestPeerID1,
|
|
}, nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) Pin(ctx context.Context, c cid.Cid, maxDepth int) error {
|
|
ipfs.pins.Store(c.String(), maxDepth)
|
|
return nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) Unpin(ctx context.Context, c cid.Cid) error {
|
|
ipfs.pins.Delete(c.String())
|
|
return nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) PinLsCid(ctx context.Context, c cid.Cid) (api.IPFSPinStatus, error) {
|
|
dI, ok := ipfs.pins.Load(c.String())
|
|
if !ok {
|
|
return api.IPFSPinStatusUnpinned, nil
|
|
}
|
|
depth := dI.(int)
|
|
if depth == 0 {
|
|
return api.IPFSPinStatusDirect, nil
|
|
}
|
|
return api.IPFSPinStatusRecursive, nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) PinLs(ctx context.Context, filter string) (map[string]api.IPFSPinStatus, error) {
|
|
m := make(map[string]api.IPFSPinStatus)
|
|
var st api.IPFSPinStatus
|
|
ipfs.pins.Range(func(k, v interface{}) bool {
|
|
switch v.(int) {
|
|
case 0:
|
|
st = api.IPFSPinStatusDirect
|
|
default:
|
|
st = api.IPFSPinStatusRecursive
|
|
}
|
|
|
|
m[k.(string)] = st
|
|
return true
|
|
})
|
|
|
|
return m, nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) SwarmPeers(ctx context.Context) (api.SwarmPeers, error) {
|
|
return []peer.ID{test.TestPeerID4, test.TestPeerID5}, nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) RepoStat(ctx context.Context) (api.IPFSRepoStat, error) {
|
|
return api.IPFSRepoStat{RepoSize: 100, StorageMax: 1000}, nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) Resolve(ctx context.Context, path string) (cid.Cid, error) {
|
|
_, err := gopath.ParsePath(path)
|
|
if err != nil {
|
|
return cid.Undef, err
|
|
}
|
|
|
|
return test.MustDecodeCid(test.TestCidResolved), nil
|
|
}
|
|
func (ipfs *mockConnector) ConnectSwarms(ctx context.Context) error { return nil }
|
|
func (ipfs *mockConnector) ConfigKey(keypath string) (interface{}, error) { return nil, nil }
|
|
|
|
func (ipfs *mockConnector) BlockPut(ctx context.Context, nwm api.NodeWithMeta) error {
|
|
ipfs.blocks.Store(nwm.Cid, nwm.Data)
|
|
return nil
|
|
}
|
|
|
|
func (ipfs *mockConnector) BlockGet(ctx context.Context, c cid.Cid) ([]byte, error) {
|
|
d, ok := ipfs.blocks.Load(c.String())
|
|
if !ok {
|
|
return nil, errors.New("block not found")
|
|
}
|
|
return d.([]byte), nil
|
|
}
|
|
|
|
type mockTracer struct {
|
|
mockComponent
|
|
}
|
|
|
|
func testingCluster(t *testing.T) (*Cluster, *mockAPI, *mockConnector, state.State, PinTracker) {
|
|
clusterCfg, _, _, _, consensusCfg, maptrackerCfg, statelesstrackerCfg, bmonCfg, psmonCfg, _, _ := testingConfigs()
|
|
|
|
host, err := NewClusterHost(context.Background(), clusterCfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
api := &mockAPI{}
|
|
proxy := &mockProxy{}
|
|
ipfs := &mockConnector{}
|
|
st := mapstate.NewMapState()
|
|
tracker := makePinTracker(t, clusterCfg.ID, maptrackerCfg, statelesstrackerCfg, clusterCfg.Peername)
|
|
tracer := &mockTracer{}
|
|
|
|
raftcon, _ := raft.NewConsensus(host, consensusCfg, st, false)
|
|
|
|
bmonCfg.CheckInterval = 2 * time.Second
|
|
psmonCfg.CheckInterval = 2 * time.Second
|
|
mon := makeMonitor(t, host, bmonCfg, psmonCfg)
|
|
|
|
alloc := ascendalloc.NewAllocator()
|
|
numpinCfg := &numpin.Config{}
|
|
numpinCfg.Default()
|
|
inf, _ := numpin.NewInformer(numpinCfg)
|
|
|
|
ReadyTimeout = consensusCfg.WaitForLeaderTimeout + 1*time.Second
|
|
|
|
cl, err := NewCluster(
|
|
host,
|
|
clusterCfg,
|
|
raftcon,
|
|
[]API{api, proxy},
|
|
ipfs,
|
|
st,
|
|
tracker,
|
|
mon,
|
|
alloc,
|
|
inf,
|
|
tracer,
|
|
)
|
|
if err != nil {
|
|
t.Fatal("cannot create cluster:", err)
|
|
}
|
|
<-cl.Ready()
|
|
return cl, api, ipfs, st, tracker
|
|
}
|
|
|
|
func cleanRaft() {
|
|
raftDirs, _ := filepath.Glob("raftFolderFromTests*")
|
|
for _, dir := range raftDirs {
|
|
os.RemoveAll(dir)
|
|
}
|
|
}
|
|
|
|
func testClusterShutdown(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
err := cl.Shutdown(ctx)
|
|
if err != nil {
|
|
t.Error("cluster shutdown failed:", err)
|
|
}
|
|
cl.Shutdown(ctx)
|
|
cl, _, _, _, _ = testingCluster(t)
|
|
err = cl.Shutdown(ctx)
|
|
if err != nil {
|
|
t.Error("cluster shutdown failed:", err)
|
|
}
|
|
}
|
|
|
|
func TestClusterStateSync(t *testing.T) {
|
|
ctx := context.Background()
|
|
cleanRaft()
|
|
cl, _, _, st, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
err := cl.StateSync(ctx)
|
|
if err == nil {
|
|
t.Fatal("expected an error as there is no state to sync")
|
|
}
|
|
|
|
c, _ := cid.Decode(test.TestCid1)
|
|
err = cl.Pin(ctx, api.PinCid(c))
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
|
|
err = cl.StateSync(ctx)
|
|
if err != nil {
|
|
t.Fatal("sync after pinning should have worked:", err)
|
|
}
|
|
|
|
// Modify state on the side so the sync does not
|
|
// happen on an empty slide
|
|
st.Rm(ctx, c)
|
|
err = cl.StateSync(ctx)
|
|
if err != nil {
|
|
t.Fatal("sync with recover should have worked:", err)
|
|
}
|
|
}
|
|
|
|
func TestClusterID(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
id := cl.ID(ctx)
|
|
if len(id.Addresses) == 0 {
|
|
t.Error("expected more addresses")
|
|
}
|
|
if id.ID == "" {
|
|
t.Error("expected a cluster ID")
|
|
}
|
|
if id.Version != version.Version.String() {
|
|
t.Error("version should match current version")
|
|
}
|
|
//if id.PublicKey == nil {
|
|
// t.Error("publicKey should not be empty")
|
|
//}
|
|
}
|
|
|
|
func TestClusterPin(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
c, _ := cid.Decode(test.TestCid1)
|
|
err := cl.Pin(ctx, api.PinCid(c))
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
|
|
// test an error case
|
|
cl.consensus.Shutdown(ctx)
|
|
pin := api.PinCid(c)
|
|
pin.ReplicationFactorMax = 1
|
|
pin.ReplicationFactorMin = 1
|
|
err = cl.Pin(ctx, pin)
|
|
if err == nil {
|
|
t.Error("expected an error but things worked")
|
|
}
|
|
}
|
|
|
|
func TestClusterPinPath(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
pin, err := cl.PinPath(ctx, api.PinPath{Path: test.TestPathIPFS2})
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
if pin.Cid.String() != test.TestCidResolved {
|
|
t.Error("expected a different cid, found", pin.Cid.String())
|
|
}
|
|
|
|
// test an error case
|
|
_, err = cl.PinPath(ctx, api.PinPath{Path: test.TestInvalidPath1})
|
|
if err == nil {
|
|
t.Error("expected an error but things worked")
|
|
}
|
|
}
|
|
|
|
func TestAddFile(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
t.Run("local", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.Shard = false
|
|
params.Name = "testlocal"
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
c, err := cl.AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if c.String() != test.ShardingDirBalancedRootCID {
|
|
t.Fatal("unexpected root CID for local add")
|
|
}
|
|
|
|
pinDelay()
|
|
|
|
pin := cl.StatusLocal(ctx, c)
|
|
if pin.Error != "" {
|
|
t.Fatal(pin.Error)
|
|
}
|
|
if pin.Status != api.TrackerStatusPinned {
|
|
t.Error("cid should be pinned")
|
|
}
|
|
|
|
cl.Unpin(ctx, c) // unpin so we can pin the shard in next test
|
|
pinDelay()
|
|
})
|
|
|
|
t.Run("shard", func(t *testing.T) {
|
|
params := api.DefaultAddParams()
|
|
params.Shard = true
|
|
params.Name = "testshard"
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
c, err := cl.AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if c.String() != test.ShardingDirBalancedRootCID {
|
|
t.Fatal("unexpected root CID for local add")
|
|
}
|
|
|
|
pinDelay()
|
|
|
|
// We know that this produces 14 shards.
|
|
sharding.VerifyShards(t, c, cl, cl.ipfs, 14)
|
|
})
|
|
}
|
|
|
|
func TestUnpinShard(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
sth := test.NewShardingTestHelper()
|
|
defer sth.Clean(t)
|
|
|
|
params := api.DefaultAddParams()
|
|
params.Shard = true
|
|
params.Name = "testshard"
|
|
mfr, closer := sth.GetTreeMultiReader(t)
|
|
defer closer.Close()
|
|
r := multipart.NewReader(mfr, mfr.Boundary())
|
|
root, err := cl.AddFile(r, params)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
pinDelay()
|
|
|
|
// We know that this produces 14 shards.
|
|
sharding.VerifyShards(t, root, cl, cl.ipfs, 14)
|
|
|
|
// skipping errors, VerifyShards has checked
|
|
pinnedCids := []cid.Cid{}
|
|
pinnedCids = append(pinnedCids, root)
|
|
metaPin, _ := cl.PinGet(ctx, root)
|
|
cDag, _ := cl.PinGet(ctx, metaPin.Reference)
|
|
pinnedCids = append(pinnedCids, cDag.Cid)
|
|
cDagBlock, _ := cl.ipfs.BlockGet(ctx, cDag.Cid)
|
|
cDagNode, _ := sharding.CborDataToNode(cDagBlock, "cbor")
|
|
for _, l := range cDagNode.Links() {
|
|
pinnedCids = append(pinnedCids, l.Cid)
|
|
}
|
|
|
|
t.Run("unpin clusterdag should fail", func(t *testing.T) {
|
|
err := cl.Unpin(ctx, cDag.Cid)
|
|
if err == nil {
|
|
t.Fatal("should not allow unpinning the cluster DAG directly")
|
|
}
|
|
t.Log(err)
|
|
})
|
|
|
|
t.Run("unpin shard should fail", func(t *testing.T) {
|
|
err := cl.Unpin(ctx, cDagNode.Links()[0].Cid)
|
|
if err == nil {
|
|
t.Fatal("should not allow unpinning shards directly")
|
|
}
|
|
t.Log(err)
|
|
})
|
|
|
|
t.Run("normal unpin", func(t *testing.T) {
|
|
err := cl.Unpin(ctx, root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
pinDelay()
|
|
|
|
for _, c := range pinnedCids {
|
|
st := cl.StatusLocal(ctx, c)
|
|
if st.Status != api.TrackerStatusUnpinned {
|
|
t.Errorf("%s should have been unpinned but is %s", c, st.Status)
|
|
}
|
|
|
|
st2, err := cl.ipfs.PinLsCid(context.Background(), c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st2 != api.IPFSPinStatusUnpinned {
|
|
t.Errorf("%s should have been unpinned in ipfs but is %d", c, st2)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// func singleShardedPin(t *testing.T, cl *Cluster) {
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// cCdag, _ := cid.Decode(test.TestCdagCid)
|
|
// cMeta, _ := cid.Decode(test.TestMetaRootCid)
|
|
// pinMeta(t, cl, []cid.Cid{cShard}, cCdag, cMeta)
|
|
// }
|
|
|
|
// func pinMeta(t *testing.T, cl *Cluster, shardCids []cid.Cid, cCdag, cMeta cid.Cid) {
|
|
// for _, cShard := range shardCids {
|
|
// shardPin := api.Pin{
|
|
// Cid: cShard,
|
|
// Type: api.ShardType,
|
|
// MaxDepth: 1,
|
|
// PinOptions: api.PinOptions{
|
|
// ReplicationFactorMin: -1,
|
|
// ReplicationFactorMax: -1,
|
|
// },
|
|
// }
|
|
// err := cl.Pin(shardPin)
|
|
// if err != nil {
|
|
// t.Fatal("shard pin should have worked:", err)
|
|
// }
|
|
// }
|
|
|
|
// parents := cid.NewSet()
|
|
// parents.Add(cMeta)
|
|
// cdagPin := api.Pin{
|
|
// Cid: cCdag,
|
|
// Type: api.ClusterDAGType,
|
|
// MaxDepth: 0,
|
|
// PinOptions: api.PinOptions{
|
|
// ReplicationFactorMin: -1,
|
|
// ReplicationFactorMax: -1,
|
|
// },
|
|
// }
|
|
// err := cl.Pin(cdagPin)
|
|
// if err != nil {
|
|
// t.Fatal("pin should have worked:", err)
|
|
// }
|
|
|
|
// metaPin := api.Pin{
|
|
// Cid: cMeta,
|
|
// Type: api.MetaType,
|
|
// Clusterdag: cCdag,
|
|
// }
|
|
// err = cl.Pin(metaPin)
|
|
// if err != nil {
|
|
// t.Fatal("pin should have worked:", err)
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterPinMeta(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// singleShardedPin(t, cl)
|
|
// }
|
|
|
|
// func TestClusterUnpinShardFail(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// singleShardedPin(t, cl)
|
|
// // verify pins
|
|
// if len(cl.Pins()) != 3 {
|
|
// t.Fatal("should have 3 pins")
|
|
// }
|
|
// // Unpinning metadata should fail
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// cCdag, _ := cid.Decode(test.TestCdagCid)
|
|
|
|
// err := cl.Unpin(cShard)
|
|
// if err == nil {
|
|
// t.Error("should error when unpinning shard")
|
|
// }
|
|
// err = cl.Unpin(cCdag)
|
|
// if err == nil {
|
|
// t.Error("should error when unpinning cluster dag")
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterUnpinMeta(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// singleShardedPin(t, cl)
|
|
// // verify pins
|
|
// if len(cl.Pins()) != 3 {
|
|
// t.Fatal("should have 3 pins")
|
|
// }
|
|
// // Unpinning from root should work
|
|
// cMeta, _ := cid.Decode(test.TestMetaRootCid)
|
|
|
|
// err := cl.Unpin(cMeta)
|
|
// if err != nil {
|
|
// t.Error(err)
|
|
// }
|
|
// }
|
|
|
|
// func pinTwoParentsOneShard(t *testing.T, cl *Cluster) {
|
|
// singleShardedPin(t, cl)
|
|
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// cShard2, _ := cid.Decode(test.TestShardCid2)
|
|
// cCdag2, _ := cid.Decode(test.TestCdagCid2)
|
|
// cMeta2, _ := cid.Decode(test.TestMetaRootCid2)
|
|
// pinMeta(t, cl, []cid.Cid{cShard, cShard2}, cCdag2, cMeta2)
|
|
|
|
// shardPin, err := cl.PinGet(cShard)
|
|
// if err != nil {
|
|
// t.Fatal("pin should be in state")
|
|
// }
|
|
// if shardPin.Parents.Len() != 2 {
|
|
// t.Fatal("unexpected parent set in shared shard")
|
|
// }
|
|
|
|
// shardPin2, err := cl.PinGet(cShard2)
|
|
// if shardPin2.Parents.Len() != 1 {
|
|
// t.Fatal("unexpected parent set in unshared shard")
|
|
// }
|
|
// if err != nil {
|
|
// t.Fatal("pin should be in state")
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterPinShardTwoParents(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// pinTwoParentsOneShard(t, cl)
|
|
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// shardPin, err := cl.PinGet(cShard)
|
|
// if err != nil {
|
|
// t.Fatal("double pinned shard should be pinned")
|
|
// }
|
|
// if shardPin.Parents == nil || shardPin.Parents.Len() != 2 {
|
|
// t.Fatal("double pinned shard should have two parents")
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterUnpinShardSecondParent(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// pinTwoParentsOneShard(t, cl)
|
|
// if len(cl.Pins()) != 6 {
|
|
// t.Fatal("should have 6 pins")
|
|
// }
|
|
// cMeta2, _ := cid.Decode(test.TestMetaRootCid2)
|
|
// err := cl.Unpin(cMeta2)
|
|
// if err != nil {
|
|
// t.Error(err)
|
|
// }
|
|
|
|
// pinDelay()
|
|
|
|
// if len(cl.Pins()) != 3 {
|
|
// t.Fatal("should have 3 pins")
|
|
// }
|
|
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// cCdag, _ := cid.Decode(test.TestCdagCid)
|
|
// shardPin, err := cl.PinGet(cShard)
|
|
// if err != nil {
|
|
// t.Fatal("double pinned shard node should still be pinned")
|
|
// }
|
|
// if shardPin.Parents == nil || shardPin.Parents.Len() != 1 ||
|
|
// !shardPin.Parents.Has(cCdag) {
|
|
// t.Fatalf("shard node should have single original parent %v", shardPin.Parents.Keys())
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterUnpinShardFirstParent(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// pinTwoParentsOneShard(t, cl)
|
|
// if len(cl.Pins()) != 6 {
|
|
// t.Fatal("should have 6 pins")
|
|
// }
|
|
|
|
// cMeta, _ := cid.Decode(test.TestMetaRootCid)
|
|
// err := cl.Unpin(cMeta)
|
|
// if err != nil {
|
|
// t.Error(err)
|
|
// }
|
|
// if len(cl.Pins()) != 4 {
|
|
// t.Fatal("should have 4 pins")
|
|
// }
|
|
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// cShard2, _ := cid.Decode(test.TestShardCid2)
|
|
// cCdag2, _ := cid.Decode(test.TestCdagCid2)
|
|
// shardPin, err := cl.PinGet(cShard)
|
|
// if err != nil {
|
|
// t.Fatal("double pinned shard node should still be pinned")
|
|
// }
|
|
// if shardPin.Parents == nil || shardPin.Parents.Len() != 1 ||
|
|
// !shardPin.Parents.Has(cCdag2) {
|
|
// t.Fatal("shard node should have single original parent")
|
|
// }
|
|
// _, err = cl.PinGet(cShard2)
|
|
// if err != nil {
|
|
// t.Fatal("other shard shoud still be pinned too")
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterPinTwoMethodsFail(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// // First pin normally then sharding pin fails
|
|
// c, _ := cid.Decode(test.TestMetaRootCid)
|
|
// err := cl.Pin(api.PinCid(c))
|
|
// if err != nil {
|
|
// t.Fatal("pin should have worked:", err)
|
|
// }
|
|
|
|
// cCdag, _ := cid.Decode(test.TestCdagCid)
|
|
// cMeta, _ := cid.Decode(test.TestMetaRootCid)
|
|
// metaPin := api.Pin{
|
|
// Cid: cMeta,
|
|
// Type: api.MetaType,
|
|
// Clusterdag: cCdag,
|
|
// }
|
|
// err = cl.Pin(metaPin)
|
|
// if err == nil {
|
|
// t.Fatal("pin should have failed:", err)
|
|
// }
|
|
|
|
// err = cl.Unpin(c)
|
|
// if err != nil {
|
|
// t.Fatal("unpin should have worked:", err)
|
|
// }
|
|
|
|
// singleShardedPin(t, cl)
|
|
// err = cl.Pin(api.PinCid(c))
|
|
// if err == nil {
|
|
// t.Fatal("pin should have failed:", err)
|
|
// }
|
|
// }
|
|
|
|
// func TestClusterRePinShard(t *testing.T) {
|
|
// cl, _, _, _, _ := testingCluster(t)
|
|
// defer cleanRaft()
|
|
// defer cl.Shutdown()
|
|
|
|
// cCdag, _ := cid.Decode(test.TestCdagCid)
|
|
// cShard, _ := cid.Decode(test.TestShardCid)
|
|
// shardPin := api.Pin{
|
|
// Cid: cShard,
|
|
// Type: api.ShardType,
|
|
// ReplicationFactorMin: -1,
|
|
// ReplicationFactorMax: -1,
|
|
// Recursive: true,
|
|
// }
|
|
// err := cl.Pin(shardPin)
|
|
// if err != nil {
|
|
// t.Fatal("shard pin should have worked:", err)
|
|
// }
|
|
|
|
// parents := cid.NewSet()
|
|
// parents.Add(cCdag)
|
|
// shardPin.Parents = parents
|
|
// err = cl.Pin(shardPin)
|
|
// if err != nil {
|
|
// t.Fatal("repinning shard pin with different parents should have worked:", err)
|
|
// }
|
|
|
|
// shardPin.ReplicationFactorMin = 3
|
|
// shardPin.ReplicationFactorMax = 5
|
|
// err = cl.Pin(shardPin)
|
|
// if err == nil {
|
|
// t.Fatal("repinning shard pin with different repl factors should have failed:", err)
|
|
// }
|
|
// }
|
|
|
|
func TestClusterPins(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
c, _ := cid.Decode(test.TestCid1)
|
|
err := cl.Pin(ctx, api.PinCid(c))
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
|
|
pins := cl.Pins(ctx)
|
|
if len(pins) != 1 {
|
|
t.Fatal("pin should be part of the state")
|
|
}
|
|
if !pins[0].Cid.Equals(c) || pins[0].ReplicationFactorMin != -1 || pins[0].ReplicationFactorMax != -1 {
|
|
t.Error("the Pin does not look as expected")
|
|
}
|
|
}
|
|
|
|
func TestClusterPinGet(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
c, _ := cid.Decode(test.TestCid1)
|
|
err := cl.Pin(ctx, api.PinCid(c))
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
|
|
pin, err := cl.PinGet(ctx, c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !pin.Cid.Equals(c) || pin.ReplicationFactorMin != -1 || pin.ReplicationFactorMax != -1 {
|
|
t.Error("the Pin does not look as expected")
|
|
}
|
|
|
|
c2, _ := cid.Decode(test.TestCid2)
|
|
_, err = cl.PinGet(ctx, c2)
|
|
if err == nil {
|
|
t.Fatal("expected an error")
|
|
}
|
|
}
|
|
|
|
func TestClusterUnpin(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
c, _ := cid.Decode(test.TestCid1)
|
|
// Unpin should error without pin being committed to state
|
|
err := cl.Unpin(ctx, c)
|
|
if err == nil {
|
|
t.Error("unpin should have failed")
|
|
}
|
|
|
|
// Unpin after pin should succeed
|
|
err = cl.Pin(ctx, api.PinCid(c))
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
err = cl.Unpin(ctx, c)
|
|
if err != nil {
|
|
t.Error("unpin should have worked:", err)
|
|
}
|
|
|
|
// test another error case
|
|
cl.consensus.Shutdown(ctx)
|
|
err = cl.Unpin(ctx, c)
|
|
if err == nil {
|
|
t.Error("expected an error but things worked")
|
|
}
|
|
}
|
|
|
|
func TestClusterUnpinPath(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
// Unpin should error without pin being committed to state
|
|
_, err := cl.UnpinPath(ctx, test.TestPathIPFS2)
|
|
if err == nil {
|
|
t.Error("unpin with path should have failed")
|
|
}
|
|
|
|
// Unpin after pin should succeed
|
|
pin, err := cl.PinPath(ctx, api.PinPath{Path: test.TestPathIPFS2})
|
|
if err != nil {
|
|
t.Fatal("pin with should have worked:", err)
|
|
}
|
|
if pin.Cid.String() != test.TestCidResolved {
|
|
t.Error("expected a different cid, found", pin.Cid.String())
|
|
}
|
|
|
|
pin, err = cl.UnpinPath(ctx, test.TestPathIPFS2)
|
|
if err != nil {
|
|
t.Error("unpin with path should have worked:", err)
|
|
}
|
|
if pin.Cid.String() != test.TestCidResolved {
|
|
t.Error("expected a different cid, found", pin.Cid.String())
|
|
}
|
|
}
|
|
|
|
func TestClusterPeers(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
peers := cl.Peers(ctx)
|
|
if len(peers) != 1 {
|
|
t.Fatal("expected 1 peer")
|
|
}
|
|
|
|
clusterCfg := &Config{}
|
|
clusterCfg.LoadJSON(testingClusterCfg)
|
|
if peers[0].ID != clusterCfg.ID {
|
|
t.Error("bad member")
|
|
}
|
|
}
|
|
|
|
func TestVersion(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
if cl.Version() != version.Version.String() {
|
|
t.Error("bad Version()")
|
|
}
|
|
}
|
|
|
|
func TestClusterRecoverAllLocal(t *testing.T) {
|
|
ctx := context.Background()
|
|
cl, _, _, _, _ := testingCluster(t)
|
|
defer cleanRaft()
|
|
defer cl.Shutdown(ctx)
|
|
|
|
c, _ := cid.Decode(test.TestCid1)
|
|
err := cl.Pin(ctx, api.PinCid(c))
|
|
if err != nil {
|
|
t.Fatal("pin should have worked:", err)
|
|
}
|
|
|
|
pinDelay()
|
|
|
|
recov, err := cl.RecoverAllLocal(ctx)
|
|
if err != nil {
|
|
t.Error("did not expect an error")
|
|
}
|
|
if len(recov) != 1 {
|
|
t.Fatalf("there should be only one pin, got = %d", len(recov))
|
|
}
|
|
if recov[0].Status != api.TrackerStatusPinned {
|
|
t.Errorf("the pin should have been recovered, got = %v", recov[0].Status)
|
|
}
|
|
}
|