From 5fb2b6ae950d438cb01add5da0330dbbfc03f78b Mon Sep 17 00:00:00 2001 From: Kishan Mohanbhai Sagathiya Date: Wed, 20 Feb 2019 11:07:50 +0000 Subject: [PATCH] Add PinPath/UnpinPath support. Squashed commit of the following: commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e Merge: d125f69 aaada42 Author: Hector Sanjuan Date: Wed Feb 20 11:02:00 2019 +0000 Merge pull request #634 from ipfs/issue_450 Support PinPath, UnpinPath (resolve before pinning) commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8 Author: Kishan Mohanbhai Sagathiya Date: Tue Feb 19 22:16:25 2019 +0530 formatResponse accepts api.Pin and not api.PinSerial commit b5da4bea045865814cc422da71827b44ddd44b90 Merge: ba59036 cc8dd7e Author: Kishan Mohanbhai Sagathiya Date: Tue Feb 19 21:36:46 2019 +0530 Merge branch 'master' into issue_450 commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f Merge: f002914 d59880c Author: Kishan Mohanbhai Sagathiya Date: Mon Feb 18 08:41:11 2019 +0530 Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450 commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec Author: Kishan Mohanbhai Sagathiya Date: Mon Feb 18 08:31:39 2019 +0530 PinPath: more improvements Added tracing for new methods commit d59880c338eaa8214fe06b4f930a540793d78407 Merge: 0ca4c7c b4f0eb3 Author: Hector Sanjuan Date: Wed Feb 13 15:22:49 2019 +0000 Merge branch 'master' into issue_450 commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030 Merge: d35017a ecef9ea Author: Kishan Mohanbhai Sagathiya Date: Tue Feb 12 13:10:13 2019 +0530 Merge branch 'master' into issue_450 commit d35017a8de91ca9fc9a9a047c48c75134cee9f98 Author: Kishan Mohanbhai Sagathiya Date: Tue Feb 12 13:07:25 2019 +0530 PinPath: more improvements - Worth having `PinOptions` as a separate field in the struct and constructing the query in the test with ToQuery() - sharness: "intialization" line can be placed outside the tests at the top commit 68e3b90417ffbad89d41a70ac81d85f9037f8848 Author: Kishan Mohanbhai Sagathiya Date: Sun Feb 10 21:43:50 2019 +0530 Using if-continue pattern instead of if-else License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 3c29799f3b85be328b27508332ab92049d8b82f3 Merge: 956790b 4324889 Author: Kishan Mohanbhai Sagathiya Date: Thu Feb 7 10:25:52 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 956790b381db9858e4194f983e898b07dc51ba66 Author: Kishan Mohanbhai Sagathiya Date: Wed Feb 6 21:11:20 2019 +0530 Removing resolved path License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 7191cc46cedfbec116a9746937e28881b50ca044 Author: Kishan Mohanbhai Sagathiya Date: Wed Feb 6 16:45:07 2019 +0530 Fix go vet License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4 Author: Kishan Mohanbhai Sagathiya Date: Wed Feb 6 16:07:03 2019 +0530 Fixed linting error License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 23c57eb467755a1f21387a1615a7f34e97348053 Author: Kishan Mohanbhai Sagathiya Date: Wed Feb 6 09:20:41 2019 +0530 Fixed tests License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4 Merge: 17e555e 5a7ee1d Author: Kishan Mohanbhai Sagathiya Date: Wed Feb 6 00:07:10 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 17e555e4a7c574413df90aac70c5cc29cab98f54 Author: Hector Sanjuan Date: Tue Feb 5 16:58:50 2019 +0000 PinPath: address some feedback + improvements * Changed client's Pin() API and PinPath to be consistent * Added helper methods to turn PinPath to query and back * Make code and tests build * Use TestCidResolved everywhere * Fix cluster.PinPath arguments * Fix formatting of responses with --no-status * Make tests readable and call Fatal when needed * Use a pathTestCases variable commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38 Author: Kishan Mohanbhai Sagathiya Date: Tue Feb 5 18:34:26 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) Addressed review comments as in https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit a8b4f181d2d7afed32ee41331dfaab19fd66a173 Author: Kishan Mohanbhai Sagathiya Date: Tue Jan 29 22:41:27 2019 +0530 Fixing tests License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit e39b95ca19e4d75506f4f492678245ef13936a44 Author: Kishan Mohanbhai Sagathiya Date: Tue Jan 29 14:52:53 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - PinPath and UnpinPath should return api.Pin - PinPath should accept pin options - Removing duplicate logic for Resolve from cluster - And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit d146075126320896665ba58d337a13789f68ea86 Author: Kishan Mohanbhai Sagathiya Date: Wed Jan 23 17:08:41 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) PinPath(in both rest and rpc) should return a serializable struct in the form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid" to marshal and unmarshal) License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 1f4869568a8adb450275257154ea3a26d03a30f3 Merge: 7acfd28 a244af9 Author: Kishan Mohanbhai Sagathiya Date: Wed Jan 23 07:18:56 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4 Author: Kishan Mohanbhai Sagathiya Date: Tue Jan 22 18:14:32 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - RPC must always use serializable structs - In command, just use pin with path as cid is also a valid path - Addressing many other small review comments as in https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62 Author: Kishan Mohanbhai Sagathiya Date: Wed Jan 16 09:36:42 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) Extra logic for path checking should go into resolve so that it can be properly reused Added sharness tests License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 9116bda3534e77bb391d873051bb520a1b01a326 Author: Kishan Mohanbhai Sagathiya Date: Wed Jan 16 08:08:07 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) error strings should not be capitalized Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit ca7e61861374f456300a85ddc0374e594f74f963 Author: Kishan Mohanbhai Sagathiya Date: Tue Jan 15 23:40:25 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) Tests Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 522fbcd899f01c01680375561a32a87464157c0a Merge: f1a56ab f7bc468 Author: Kishan Mohanbhai Sagathiya Date: Tue Jan 15 10:40:54 2019 +0530 Merge branch 'master' into issue_450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit f1a56ab925fb74c0c44273a4524afa4843cf757f Author: Kishan Mohanbhai Sagathiya Date: Mon Jan 14 20:58:17 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - IPFS Connector should act as a pure IPFS client, any extra logic should go to cluster.go - Use cid.Undef, instead of cid.Cid{} Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c Author: Kishan Mohanbhai Sagathiya Date: Thu Jan 10 08:57:17 2019 +0530 Support PinPath, UnpinPath(resolve before pinning) - Separate handlers, methods and rpc apis for PinPath and UnpinPath from Pin and Unpin - Support ipld paths as well Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c Merge: 91ceb47 21170c4 Author: Kishan Mohanbhai Sagathiya Date: Wed Jan 9 19:38:35 2019 +0530 Merge branch 'issue_450_old' into HEAD License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 91ceb4796259ca7ef2974ec43e6a278a12796b13 Author: Kishan Mohanbhai Sagathiya Date: Wed Jan 9 19:36:41 2019 +0530 Revert "WIP: Figure out why test does not impleme" This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75. License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya commit 28a3a3f25dce6f296c8cbef86221644c099a7e75 Author: cd10012 Date: Tue Jul 24 23:23:10 2018 -0400 WIP: Figure out why test does not implement IPFSConnector interface... License: MIT Signed-off-by: cd10012 commit 21170c48e77e69583db64544b08120a9baf40d8d Author: Kishan Mohanbhai Sagathiya Date: Tue Jan 8 10:37:59 2019 +0530 Support PinPath, UnpinPath (resolve before pinning) This commit adds API support for pinning using path `POST /pins/` and `DELETE /pins/` will resolve the path into a cid and perform perform pinning or unpinning Fixes #450 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya Co-authored-by: Hector Sanjuan License: MIT Signed-off-by: Hector Sanjuan --- api/rest/client/client.go | 8 +- api/rest/client/methods.go | 53 +++++++++++-- api/rest/client/methods_test.go | 103 +++++++++++++++++++++++++- api/rest/restapi.go | 112 ++++++++++++++++++---------- api/rest/restapi_test.go | 97 ++++++++++++++++++++++++ api/types.go | 44 +++++++++++ cluster.go | 115 +++++++++++++++++++---------- cluster_test.go | 61 +++++++++++++++ cmd/ipfs-cluster-ctl/main.go | 33 ++++----- ipfscluster.go | 2 + ipfsconn/ipfshttp/ipfshttp.go | 42 +++++++++++ ipfsconn/ipfshttp/ipfshttp_test.go | 15 ++++ rpc_api.go | 14 ++++ sharness/t0030-ctl-pin.sh | 29 ++++++++ test/cids.go | 25 +++++-- test/ipfs_mock.go | 2 + test/rpc_api_mock.go | 23 +++++- 17 files changed, 670 insertions(+), 108 deletions(-) diff --git a/api/rest/client/client.go b/api/rest/client/client.go index bc0eb243..68502f1e 100644 --- a/api/rest/client/client.go +++ b/api/rest/client/client.go @@ -63,10 +63,16 @@ type Client interface { // Pin tracks a Cid with the given replication factor and a name for // human-friendliness. - Pin(ctx context.Context, ci cid.Cid, replicationFactorMin, replicationFactorMax int, name string) error + Pin(ctx context.Context, ci cid.Cid, opts api.PinOptions) error // Unpin untracks a Cid from cluster. Unpin(ctx context.Context, ci cid.Cid) error + // PinPath resolves given path into a cid and performs the pin operation. + PinPath(ctx context.Context, path string, opts api.PinOptions) (api.Pin, error) + // UnpinPath resolves given path into a cid and performs the unpin operation. + // It returns api.Pin of the given cid before it is unpinned. + UnpinPath(ctx context.Context, path string) (api.Pin, error) + // Allocations returns the consensus state listing all tracked items // and the peers that should be pinning them. Allocations(ctx context.Context, filter api.PinType) ([]api.Pin, error) diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index ef76a91d..9983a4a5 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -19,6 +19,7 @@ import ( cid "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" + gopath "github.com/ipfs/go-path" peer "github.com/libp2p/go-libp2p-peer" ) @@ -77,20 +78,17 @@ func (c *defaultClient) PeerRm(ctx context.Context, id peer.ID) error { // Pin tracks a Cid with the given replication factor and a name for // human-friendliness. -func (c *defaultClient) Pin(ctx context.Context, ci cid.Cid, replicationFactorMin, replicationFactorMax int, name string) error { +func (c *defaultClient) Pin(ctx context.Context, ci cid.Cid, opts api.PinOptions) error { ctx, span := trace.StartSpan(ctx, "client/Pin") defer span.End() - escName := url.QueryEscape(name) err := c.do( ctx, "POST", fmt.Sprintf( - "/pins/%s?replication-min=%d&replication-max=%d&name=%s", + "/pins/%s?%s", ci.String(), - replicationFactorMin, - replicationFactorMax, - escName, + opts.ToQuery(), ), nil, nil, @@ -106,6 +104,49 @@ func (c *defaultClient) Unpin(ctx context.Context, ci cid.Cid) error { return c.do(ctx, "DELETE", fmt.Sprintf("/pins/%s", ci.String()), nil, nil, nil) } +// PinPath allows to pin an element by the given IPFS path. +func (c *defaultClient) PinPath(ctx context.Context, path string, opts api.PinOptions) (api.Pin, error) { + ctx, span := trace.StartSpan(ctx, "client/PinPath") + defer span.End() + + var pin api.PinSerial + ipfspath, err := gopath.ParsePath(path) + if err != nil { + return api.Pin{}, err + } + + err = c.do( + ctx, + "POST", + fmt.Sprintf( + "/pins%s?%s", + ipfspath.String(), + opts.ToQuery(), + ), + nil, + nil, + &pin, + ) + + return pin.ToPin(), err +} + +// UnpinPath allows to unpin an item by providing its IPFS path. +// It returns the unpinned api.Pin information of the resolved Cid. +func (c *defaultClient) UnpinPath(ctx context.Context, p string) (api.Pin, error) { + ctx, span := trace.StartSpan(ctx, "client/UnpinPath") + defer span.End() + + var pin api.PinSerial + ipfspath, err := gopath.ParsePath(p) + if err != nil { + return api.Pin{}, err + } + + err = c.do(ctx, "DELETE", fmt.Sprintf("/pins%s", ipfspath.String()), nil, nil, &pin) + return pin.ToPin(), err +} + // Allocations returns the consensus state listing all tracked items and // the peers that should be pinning them. func (c *defaultClient) Allocations(ctx context.Context, filter api.PinType) ([]api.Pin, error) { diff --git a/api/rest/client/methods_test.go b/api/rest/client/methods_test.go index cb3c6452..66e0c0d5 100644 --- a/api/rest/client/methods_test.go +++ b/api/rest/client/methods_test.go @@ -144,7 +144,12 @@ func TestPin(t *testing.T) { testF := func(t *testing.T, c Client) { ci, _ := cid.Decode(test.TestCid1) - err := c.Pin(ctx, ci, 6, 7, "hello there") + opts := types.PinOptions{ + ReplicationFactorMin: 6, + ReplicationFactorMax: 7, + Name: "hello there", + } + err := c.Pin(ctx, ci, opts) if err != nil { t.Fatal(err) } @@ -169,6 +174,100 @@ func TestUnpin(t *testing.T) { testClients(t, api, testF) } +type pathCase struct { + path string + wantErr bool +} + +var pathTestCases = []pathCase{ + { + test.TestCidResolved, + false, + }, + { + test.TestPathIPFS1, + false, + }, + { + test.TestPathIPFS2, + false, + }, + { + test.TestPathIPNS1, + false, + }, + { + test.TestPathIPLD1, + false, + }, + { + test.TestInvalidPath1, + true, + }, +} + +func TestPinPath(t *testing.T) { + ctx := context.Background() + api := testAPI(t) + defer shutdown(api) + + opts := types.PinOptions{ + ReplicationFactorMin: 6, + ReplicationFactorMax: 7, + Name: "hello there", + } + + resultantPin := types.PinWithOpts(test.MustDecodeCid(test.TestCidResolved), opts) + + testF := func(t *testing.T, c Client) { + + for _, testCase := range pathTestCases { + p := testCase.path + pin, err := c.PinPath(ctx, p, opts) + if err != nil { + if testCase.wantErr { + continue + } + t.Fatalf("unexpected error %s: %s", p, err) + } + + if !pin.Equals(resultantPin) { + t.Errorf("expected different pin: %s", p) + t.Errorf("expected: %+v", resultantPin.ToSerial()) + t.Errorf("actual: %+v", pin.ToSerial()) + } + + } + } + + testClients(t, api, testF) +} + +func TestUnpinPath(t *testing.T) { + ctx := context.Background() + api := testAPI(t) + defer shutdown(api) + + testF := func(t *testing.T, c Client) { + for _, testCase := range pathTestCases { + p := testCase.path + pin, err := c.UnpinPath(ctx, p) + if err != nil { + if testCase.wantErr { + continue + } + t.Fatalf("unepected error %s: %s", p, err) + } + + if pin.Cid.String() != test.TestCidResolved { + t.Errorf("bad resolved Cid: %s, %s", p, pin.Cid) + } + } + } + + testClients(t, api, testF) +} + func TestAllocations(t *testing.T) { ctx := context.Background() api := testAPI(t) @@ -488,7 +587,7 @@ func TestWaitFor(t *testing.T) { } } }() - err := c.Pin(ctx, ci, 0, 0, "test") + err := c.Pin(ctx, ci, types.PinOptions{ReplicationFactorMin: 0, ReplicationFactorMax: 0, Name: "test", ShardSize: 0}) if err != nil { t.Fatal(err) } diff --git a/api/rest/restapi.go b/api/rest/restapi.go index df45a26a..5993b9b3 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -16,12 +16,12 @@ import ( "math/rand" "net" "net/http" - "strconv" "strings" "sync" "time" "github.com/rs/cors" + "go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp/propagation/tracecontext" "go.opencensus.io/trace" @@ -34,6 +34,7 @@ import ( p2phttp "github.com/hsanjuan/go-libp2p-http" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + gopath "github.com/ipfs/go-path" libp2p "github.com/libp2p/go-libp2p" rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" @@ -366,12 +367,24 @@ func (api *API) routes() []route { "/pins", api.statusAllHandler, }, + { + "Sync", + "POST", + "/pins/{hash}/sync", + api.syncHandler, + }, { "SyncAll", "POST", "/pins/sync", api.syncAllHandler, }, + { + "Recover", + "POST", + "/pins/{hash}/recover", + api.recoverHandler, + }, { "RecoverAll", "POST", @@ -390,6 +403,12 @@ func (api *API) routes() []route { "/pins/{hash}", api.pinHandler, }, + { + "PinPath", + "POST", + "/pins/{keyType:ipfs|ipns|ipld}/{path:.*}", + api.pinPathHandler, + }, { "Unpin", "DELETE", @@ -397,16 +416,10 @@ func (api *API) routes() []route { api.unpinHandler, }, { - "Sync", - "POST", - "/pins/{hash}/sync", - api.syncHandler, - }, - { - "Recover", - "POST", - "/pins/{hash}/recover", - api.recoverHandler, + "UnpinPath", + "DELETE", + "/pins/{keyType:ipfs|ipns|ipld}/{path:.*}", + api.unpinPathHandler, }, { "ConnectionGraph", @@ -688,6 +701,41 @@ func (api *API) unpinHandler(w http.ResponseWriter, r *http.Request) { } } +func (api *API) pinPathHandler(w http.ResponseWriter, r *http.Request) { + var pin types.PinSerial + if pinpath := api.parsePinPathOrError(w, r); pinpath.Path != "" { + logger.Debugf("rest api pinPathHandler: %s", pinpath.Path) + err := api.rpcClient.CallContext( + r.Context(), + "", + "Cluster", + "PinPath", + pinpath, + &pin, + ) + + api.sendResponse(w, http.StatusOK, err, pin) + logger.Debug("rest api pinPathHandler done") + } +} + +func (api *API) unpinPathHandler(w http.ResponseWriter, r *http.Request) { + var pin types.PinSerial + if pinpath := api.parsePinPathOrError(w, r); pinpath.Path != "" { + logger.Debugf("rest api unpinPathHandler: %s", pinpath.Path) + err := api.rpcClient.CallContext( + r.Context(), + "", + "Cluster", + "UnpinPath", + pinpath.Path, + &pin, + ) + api.sendResponse(w, http.StatusOK, err, pin) + logger.Debug("rest api unpinPathHandler done") + } +} + func (api *API) allocationsHandler(w http.ResponseWriter, r *http.Request) { queryValues := r.URL.Query() filterStr := queryValues.Get("filter") @@ -947,6 +995,21 @@ func (api *API) recoverHandler(w http.ResponseWriter, r *http.Request) { } } +func (api *API) parsePinPathOrError(w http.ResponseWriter, r *http.Request) types.PinPath { + vars := mux.Vars(r) + urlpath := "/" + vars["keyType"] + "/" + strings.TrimSuffix(vars["path"], "/") + + path, err := gopath.ParsePath(urlpath) + if err != nil { + api.sendResponse(w, http.StatusBadRequest, errors.New("error parsing path: "+err.Error()), nil) + return types.PinPath{} + } + + pinPath := types.PinPath{Path: path.String()} + pinPath.PinOptions.FromQuery(r.URL.Query()) + return pinPath +} + func (api *API) parseCidOrError(w http.ResponseWriter, r *http.Request) types.PinSerial { vars := mux.Vars(r) hash := vars["hash"] @@ -962,33 +1025,8 @@ func (api *API) parseCidOrError(w http.ResponseWriter, r *http.Request) types.Pi Type: uint64(types.DataType), } - queryValues := r.URL.Query() - name := queryValues.Get("name") - pin.Name = name + pin.PinOptions.FromQuery(r.URL.Query()) pin.MaxDepth = -1 // For now, all pins are recursive - rplStr := queryValues.Get("replication") - if rplStr == "" { // compat <= 0.4.0 - rplStr = queryValues.Get("replication_factor") - } - rplStrMin := queryValues.Get("replication-min") - if rplStrMin == "" { // compat <= 0.4.0 - rplStrMin = queryValues.Get("replication_factor_min") - } - rplStrMax := queryValues.Get("replication-max") - if rplStrMax == "" { // compat <= 0.4.0 - rplStrMax = queryValues.Get("replication_factor_max") - } - if rplStr != "" { // override - rplStrMin = rplStr - rplStrMax = rplStr - } - if rpl, err := strconv.Atoi(rplStrMin); err == nil { - pin.ReplicationFactorMin = rpl - } - if rpl, err := strconv.Atoi(rplStrMax); err == nil { - pin.ReplicationFactorMax = rpl - } - return pin } diff --git a/api/rest/restapi_test.go b/api/rest/restapi_test.go index 43f0f08f..9a1bb17e 100644 --- a/api/rest/restapi_test.go +++ b/api/rest/restapi_test.go @@ -584,6 +584,77 @@ func TestAPIPinEndpoint(t *testing.T) { testBothEndpoints(t, tf) } +type pathCase struct { + path string + opts api.PinOptions + wantErr bool + code int +} + +func (p *pathCase) WithQuery() string { + return p.path + "?" + p.opts.ToQuery() +} + +var testPinOpts = api.PinOptions{ + ReplicationFactorMax: 7, + ReplicationFactorMin: 6, + Name: "hello there", +} + +var pathTestCases = []pathCase{ + { + "/ipfs/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY", + testPinOpts, + false, + http.StatusOK, + }, + { + "/ipfs/QmbUNM297ZwxB8CfFAznK7H9YMesDoY6Tt5bPgt5MSCB2u/im.gif", + testPinOpts, + false, + http.StatusOK, + }, + { + "/ipfs/invalidhash", + testPinOpts, + true, + http.StatusBadRequest, + }, + // TODO: Test StatusNotFound and a case with trailing slash with paths + // test.TestPathIPNS2, test.TestPathIPLD2, test.TestInvalidPath1 +} + +func TestAPIPinEndpointWithPath(t *testing.T) { + ctx := context.Background() + rest := testAPI(t) + defer rest.Shutdown(ctx) + + resultantPin := api.PinWithOpts( + test.MustDecodeCid(test.TestCidResolved), + testPinOpts, + ) + + tf := func(t *testing.T, url urlF) { + for _, testCase := range pathTestCases { + if testCase.wantErr { + errResp := api.Error{} + makePost(t, rest, url(rest)+"/pins"+testCase.WithQuery(), []byte{}, &errResp) + if errResp.Code != testCase.code { + t.Errorf("expected different status code, expected: %d, actual: %d, path: %s\n", testCase.code, errResp.Code, testCase.path) + } + continue + } + pin := api.PinSerial{} + makePost(t, rest, url(rest)+"/pins"+testCase.WithQuery(), []byte{}, &pin) + if !pin.ToPin().Equals(resultantPin) { + t.Errorf("expected different pin,\n expected: %+v,\n actual: %+v,\n path: %s\n", resultantPin.ToSerial(), pin, testCase.path) + } + } + } + + testBothEndpoints(t, tf) +} + func TestAPIUnpinEndpoint(t *testing.T) { ctx := context.Background() rest := testAPI(t) @@ -608,6 +679,32 @@ func TestAPIUnpinEndpoint(t *testing.T) { testBothEndpoints(t, tf) } +func TestAPIUnpinEndpointWithPath(t *testing.T) { + ctx := context.Background() + rest := testAPI(t) + defer rest.Shutdown(ctx) + + tf := func(t *testing.T, url urlF) { + for _, testCase := range pathTestCases { + if testCase.wantErr { + errResp := api.Error{} + makeDelete(t, rest, url(rest)+"/pins"+testCase.path, &errResp) + if errResp.Code != testCase.code { + t.Errorf("expected different status code, expected: %d, actual: %d, path: %s\n", testCase.code, errResp.Code, testCase.path) + } + continue + } + pin := api.PinSerial{} + makeDelete(t, rest, url(rest)+"/pins"+testCase.path, &pin) + if pin.Cid != test.TestCidResolved { + t.Errorf("expected different cid, expected: %s, actual: %s, path: %s\n", test.TestCidResolved, pin.Cid, testCase.path) + } + } + } + + testBothEndpoints(t, tf) +} + func TestAPIAllocationsEndpoint(t *testing.T) { ctx := context.Background() rest := testAPI(t) diff --git a/api/types.go b/api/types.go index 433a9344..be01f157 100644 --- a/api/types.go +++ b/api/types.go @@ -12,8 +12,10 @@ import ( "bytes" "encoding/json" "fmt" + "net/url" "regexp" "sort" + "strconv" "strings" "time" @@ -710,6 +712,42 @@ type PinOptions struct { ShardSize uint64 `json:"shard_size"` } +// ToQuery returns the PinOption as query arguments. +func (po *PinOptions) ToQuery() string { + q := url.Values{} + q.Set("replication-min", fmt.Sprintf("%d", po.ReplicationFactorMin)) + q.Set("replication-max", fmt.Sprintf("%d", po.ReplicationFactorMax)) + q.Set("name", po.Name) + return q.Encode() +} + +// FromQuery is the inverse of ToQuery(). +func (po *PinOptions) FromQuery(q url.Values) { + po.Name = q.Get("name") + rplStr := q.Get("replication") + if rplStr == "" { // compat <= 0.4.0 + rplStr = q.Get("replication_factor") + } + rplStrMin := q.Get("replication-min") + if rplStrMin == "" { // compat <= 0.4.0 + rplStrMin = q.Get("replication_factor_min") + } + rplStrMax := q.Get("replication-max") + if rplStrMax == "" { // compat <= 0.4.0 + rplStrMax = q.Get("replication_factor_max") + } + if rplStr != "" { // override + rplStrMin = rplStr + rplStrMax = rplStr + } + if rpl, err := strconv.Atoi(rplStrMin); err == nil { + po.ReplicationFactorMin = rpl + } + if rpl, err := strconv.Atoi(rplStrMax); err == nil { + po.ReplicationFactorMax = rpl + } +} + // Pin carries all the information associated to a CID that is pinned // in IPFS Cluster. type Pin struct { @@ -734,6 +772,12 @@ type Pin struct { Reference cid.Cid } +// PinPath is a wrapper for holding pin options and path of the content. +type PinPath struct { + PinOptions + Path string `json:"path"` +} + // PinCid is a shortcut to create a Pin only with a Cid. Default is for pin to // be recursive and the pin to be of DataType. func PinCid(c cid.Cid) Pin { diff --git a/cluster.go b/cluster.go index 81e0951a..c7596c3d 100644 --- a/cluster.go +++ b/cluster.go @@ -381,7 +381,7 @@ func (c *Cluster) repinFromPeer(ctx context.Context, p peer.ID) { list := cState.List(ctx) for _, pin := range list { if containsPeer(pin.Allocations, p) { - ok, err := c.pin(ctx, pin, []peer.ID{p}, []peer.ID{}) // pin blacklisting this peer + _, ok, err := c.pin(ctx, pin, []peer.ID{p}, []peer.ID{}) // pin blacklisting this peer if ok && err == nil { logger.Infof("repinned %s out of %s", pin.Cid, p.Pretty()) } @@ -1020,7 +1020,7 @@ func (c *Cluster) Pin(ctx context.Context, pin api.Pin) error { _, span := trace.StartSpan(ctx, "cluster/Pin") defer span.End() ctx = trace.NewContext(c.ctx, span) - _, err := c.pin(ctx, pin, []peer.ID{}, pin.Allocations) + _, _, err := c.pin(ctx, pin, []peer.ID{}, pin.Allocations) return err } @@ -1099,24 +1099,24 @@ func (c *Cluster) setupPin(ctx context.Context, pin *api.Pin) error { } // pin performs the actual pinning and supports a blacklist to be -// able to evacuate a node and returns whether the pin was submitted +// able to evacuate a node and returns the pin object that it tried to pin, whether the pin was submitted // to the consensus layer or skipped (due to error or to the fact -// that it was already valid). -func (c *Cluster) pin(ctx context.Context, pin api.Pin, blacklist []peer.ID, prioritylist []peer.ID) (bool, error) { +// that it was already valid) and errror. +func (c *Cluster) pin(ctx context.Context, pin api.Pin, blacklist []peer.ID, prioritylist []peer.ID) (api.Pin, bool, error) { ctx, span := trace.StartSpan(ctx, "cluster/pin") defer span.End() if pin.Cid == cid.Undef { - return false, errors.New("bad pin object") + return pin, false, errors.New("bad pin object") } // setup pin might produce some side-effects to our pin err := c.setupPin(ctx, &pin) if err != nil { - return false, err + return pin, false, err } if pin.Type == api.MetaType { - return true, c.consensus.LogPin(ctx, pin) + return pin, true, c.consensus.LogPin(ctx, pin) } allocs, err := c.allocate( @@ -1128,14 +1128,14 @@ func (c *Cluster) pin(ctx context.Context, pin api.Pin, blacklist []peer.ID, pri prioritylist, ) if err != nil { - return false, err + return pin, false, err } pin.Allocations = allocs if curr, _ := c.PinGet(ctx, pin.Cid); curr.Equals(pin) { // skip pinning logger.Debugf("pinning %s skipped: already correctly allocated", pin.Cid) - return false, nil + return pin, false, nil } if len(pin.Allocations) == 0 { @@ -1144,7 +1144,39 @@ func (c *Cluster) pin(ctx context.Context, pin api.Pin, blacklist []peer.ID, pri logger.Infof("IPFS cluster pinning %s on %s:", pin.Cid, pin.Allocations) } - return true, c.consensus.LogPin(ctx, pin) + return pin, true, c.consensus.LogPin(ctx, pin) +} + +func (c *Cluster) unpin(ctx context.Context, h cid.Cid) (api.Pin, error) { + _, span := trace.StartSpan(ctx, "cluster/unpin") + defer span.End() + ctx = trace.NewContext(c.ctx, span) + + logger.Info("IPFS cluster unpinning:", h) + pin, err := c.PinGet(ctx, h) + if err != nil { + return pin, fmt.Errorf("cannot unpin pin uncommitted to state: %s", err) + } + + switch pin.Type { + case api.DataType: + return pin, c.consensus.LogUnpin(ctx, pin) + case api.ShardType: + err := "cannot unpin a shard direclty. Unpin content root CID instead." + return pin, errors.New(err) + case api.MetaType: + // Unpin cluster dag and referenced shards + err := c.unpinClusterDag(pin) + if err != nil { + return pin, err + } + return pin, c.consensus.LogUnpin(ctx, pin) + case api.ClusterDAGType: + err := "cannot unpin a Cluster DAG directly. Unpin content root CID instead." + return pin, errors.New(err) + default: + return pin, errors.New("unrecognized pin type") + } } // Unpin makes the cluster Unpin a Cid. This implies adding the Cid @@ -1157,32 +1189,8 @@ func (c *Cluster) Unpin(ctx context.Context, h cid.Cid) error { _, span := trace.StartSpan(ctx, "cluster/Unpin") defer span.End() ctx = trace.NewContext(c.ctx, span) - - logger.Info("IPFS cluster unpinning:", h) - pin, err := c.PinGet(ctx, h) - if err != nil { - return fmt.Errorf("cannot unpin pin uncommitted to state: %s", err) - } - - switch pin.Type { - case api.DataType: - return c.consensus.LogUnpin(ctx, pin) - case api.ShardType: - err := "cannot unpin a shard direclty. Unpin content root CID instead." - return errors.New(err) - case api.MetaType: - // Unpin cluster dag and referenced shards - err := c.unpinClusterDag(pin) - if err != nil { - return err - } - return c.consensus.LogUnpin(ctx, pin) - case api.ClusterDAGType: - err := "cannot unpin a Cluster DAG directly. Unpin content root CID instead." - return errors.New(err) - default: - return errors.New("unrecognized pin type") - } + _, err := c.unpin(ctx, h) + return err } // unpinClusterDag unpins the clusterDAG metadata node and the shard metadata @@ -1209,6 +1217,39 @@ func (c *Cluster) unpinClusterDag(metaPin api.Pin) error { return nil } +// PinPath pins an CID resolved from its IPFS Path. It returns the resolved +// Pin object. +func (c *Cluster) PinPath(ctx context.Context, path api.PinPath) (api.Pin, error) { + _, span := trace.StartSpan(ctx, "cluster/PinPath") + defer span.End() + + ctx = trace.NewContext(c.ctx, span) + ci, err := c.ipfs.Resolve(ctx, path.Path) + if err != nil { + return api.Pin{}, err + } + + p := api.PinCid(ci) + p.PinOptions = path.PinOptions + p, _, err = c.pin(ctx, p, []peer.ID{}, p.Allocations) + return p, err +} + +// UnpinPath unpins a CID resolved from its IPFS Path. If returns the +// previously pinned Pin object. +func (c *Cluster) UnpinPath(ctx context.Context, path string) (api.Pin, error) { + _, span := trace.StartSpan(ctx, "cluster/UnpinPath") + defer span.End() + + ctx = trace.NewContext(c.ctx, span) + ci, err := c.ipfs.Resolve(ctx, path) + if err != nil { + return api.Pin{}, err + } + + return c.unpin(ctx, ci) +} + // AddFile adds a file to the ipfs daemons of the cluster. The ipfs importer // pipeline is used to DAGify the file. Depending on input parameters this // DAG can be added locally to the calling cluster peer's ipfs repo, or diff --git a/cluster_test.go b/cluster_test.go index 63964ed5..3b1bab35 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -20,6 +20,8 @@ import ( "github.com/ipfs/ipfs-cluster/test" "github.com/ipfs/ipfs-cluster/version" + gopath "github.com/ipfs/go-path" + cid "github.com/ipfs/go-cid" rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" @@ -107,6 +109,14 @@ func (ipfs *mockConnector) RepoStat(ctx context.Context) (api.IPFSRepoStat, erro return api.IPFSRepoStat{RepoSize: 100, StorageMax: 1000}, nil } +func (ipfs *mockConnector) Resolve(ctx context.Context, path string) (cid.Cid, error) { + _, err := gopath.ParsePath(path) + if err != nil { + return cid.Undef, err + } + + return test.MustDecodeCid(test.TestCidResolved), nil +} func (ipfs *mockConnector) ConnectSwarms(ctx context.Context) error { return nil } func (ipfs *mockConnector) ConfigKey(keypath string) (interface{}, error) { return nil, nil } @@ -271,6 +281,27 @@ func TestClusterPin(t *testing.T) { } } +func TestClusterPinPath(t *testing.T) { + ctx := context.Background() + cl, _, _, _, _ := testingCluster(t) + defer cleanRaft() + defer cl.Shutdown(ctx) + + pin, err := cl.PinPath(ctx, api.PinPath{Path: test.TestPathIPFS2}) + if err != nil { + t.Fatal("pin should have worked:", err) + } + if pin.Cid.String() != test.TestCidResolved { + t.Error("expected a different cid, found", pin.Cid.String()) + } + + // test an error case + _, err = cl.PinPath(ctx, api.PinPath{Path: test.TestInvalidPath1}) + if err == nil { + t.Error("expected an error but things worked") + } +} + func TestAddFile(t *testing.T) { ctx := context.Background() cl, _, _, _, _ := testingCluster(t) @@ -772,6 +803,36 @@ func TestClusterUnpin(t *testing.T) { } } +func TestClusterUnpinPath(t *testing.T) { + ctx := context.Background() + cl, _, _, _, _ := testingCluster(t) + defer cleanRaft() + defer cl.Shutdown(ctx) + + // Unpin should error without pin being committed to state + _, err := cl.UnpinPath(ctx, test.TestPathIPFS2) + if err == nil { + t.Error("unpin with path should have failed") + } + + // Unpin after pin should succeed + pin, err := cl.PinPath(ctx, api.PinPath{Path: test.TestPathIPFS2}) + if err != nil { + t.Fatal("pin with should have worked:", err) + } + if pin.Cid.String() != test.TestCidResolved { + t.Error("expected a different cid, found", pin.Cid.String()) + } + + pin, err = cl.UnpinPath(ctx, test.TestPathIPFS2) + if err != nil { + t.Error("unpin with path should have worked:", err) + } + if pin.Cid.String() != test.TestCidResolved { + t.Error("expected a different cid, found", pin.Cid.String()) + } +} + func TestClusterPeers(t *testing.T) { ctx := context.Background() cl, _, _, _, _ := testingCluster(t) diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index a1976c0e..dcee47c5 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -512,10 +512,7 @@ peers should pin this content. }, }, Action: func(c *cli.Context) error { - cidStr := c.Args().First() - ci, err := cid.Decode(cidStr) - checkErr("parsing cid", err) - + arg := c.Args().First() rpl := c.Int("replication") rplMin := c.Int("replication-min") rplMax := c.Int("replication-max") @@ -524,16 +521,21 @@ peers should pin this content. rplMax = rpl } - cerr := globalClient.Pin(ctx, ci, rplMin, rplMax, c.String("name")) + opts := api.PinOptions{ + ReplicationFactorMin: rplMin, + ReplicationFactorMax: rplMax, + Name: c.String("name"), + } + + pin, cerr := globalClient.PinPath(ctx, arg, opts) if cerr != nil { formatResponse(c, nil, cerr) return nil } - handlePinResponseFormatFlags( ctx, c, - ci, + pin, api.TrackerStatusPinned, ) return nil @@ -567,20 +569,16 @@ although unpinning operations in the cluster may take longer or fail. }, }, Action: func(c *cli.Context) error { - - cidStr := c.Args().First() - ci, err := cid.Decode(cidStr) - checkErr("parsing cid", err) - cerr := globalClient.Unpin(ctx, ci) + arg := c.Args().First() + pin, cerr := globalClient.UnpinPath(ctx, arg) if cerr != nil { formatResponse(c, nil, cerr) return nil } - handlePinResponseFormatFlags( ctx, c, - ci, + pin, api.TrackerStatusUnpinned, ) return nil @@ -932,7 +930,7 @@ func parseCredentials(userInput string) (string, string) { func handlePinResponseFormatFlags( ctx context.Context, c *cli.Context, - ci cid.Cid, + pin api.Pin, target api.TrackerStatus, ) { @@ -940,17 +938,18 @@ func handlePinResponseFormatFlags( var cerr error if c.Bool("wait") { - status, cerr = waitFor(ci, target, c.Duration("wait-timeout")) + status, cerr = waitFor(pin.Cid, target, c.Duration("wait-timeout")) checkErr("waiting for pin status", cerr) } if c.Bool("no-status") { + formatResponse(c, pin, nil) return } if status.Cid == cid.Undef { // no status from "wait" time.Sleep(time.Second) - status, cerr = globalClient.Status(ctx, ci, false) + status, cerr = globalClient.Status(ctx, pin.Cid, false) } formatResponse(c, status, cerr) } diff --git a/ipfscluster.go b/ipfscluster.go index 7b565c95..2d77867f 100644 --- a/ipfscluster.go +++ b/ipfscluster.go @@ -83,6 +83,8 @@ type IPFSConnector interface { // RepoStat returns the current repository size and max limit as // provided by "repo stat". RepoStat(context.Context) (api.IPFSRepoStat, error) + // Resolve returns a cid given a path + Resolve(context.Context, string) (cid.Cid, error) // BlockPut directly adds a block of data to the IPFS repo BlockPut(context.Context, api.NodeWithMeta) error // BlockGet retrieves the raw data of an IPFS block diff --git a/ipfsconn/ipfshttp/ipfshttp.go b/ipfsconn/ipfshttp/ipfshttp.go index c555405d..19c15644 100644 --- a/ipfsconn/ipfshttp/ipfshttp.go +++ b/ipfsconn/ipfshttp/ipfshttp.go @@ -10,10 +10,13 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "strings" "sync" "time" + gopath "github.com/ipfs/go-path" + "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/observations" @@ -83,6 +86,10 @@ type ipfsIDResp struct { Addresses []string } +type ipfsResolveResp struct { + Path string +} + type ipfsSwarmPeersResp struct { Peers []ipfsPeer } @@ -597,6 +604,41 @@ func (ipfs *Connector) RepoStat(ctx context.Context) (api.IPFSRepoStat, error) { return stats, nil } +// Resolve accepts ipfs or ipns path and resolves it into a cid +func (ipfs *Connector) Resolve(ctx context.Context, path string) (cid.Cid, error) { + ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/Resolve") + defer span.End() + + validPath, err := gopath.ParsePath(path) + if err != nil { + logger.Error("could not parse path: " + err.Error()) + return cid.Undef, err + } + + if !strings.HasPrefix(path, "/ipns") && validPath.IsJustAKey() { + ci, _, err := gopath.SplitAbsPath(validPath) + return ci, err + } + + ctx, cancel := context.WithTimeout(ctx, ipfs.config.IPFSRequestTimeout) + defer cancel() + res, err := ipfs.postCtx(ctx, "resolve?arg="+url.QueryEscape(path), "", nil) + if err != nil { + logger.Error(err) + return cid.Undef, err + } + + var resp ipfsResolveResp + err = json.Unmarshal(res, &resp) + if err != nil { + logger.Error("could not unmarshal response: " + err.Error()) + return cid.Undef, err + } + + ci, _, err := gopath.SplitAbsPath(gopath.FromString(resp.Path)) + return ci, err +} + // SwarmPeers returns the peers currently connected to this ipfs daemon. func (ipfs *Connector) SwarmPeers(ctx context.Context) (api.SwarmPeers, error) { ctx, span := trace.StartSpan(ctx, "ipfsconn/ipfshttp/SwarmPeers") diff --git a/ipfsconn/ipfshttp/ipfshttp_test.go b/ipfsconn/ipfshttp/ipfshttp_test.go index 456b192c..3eeb57c9 100644 --- a/ipfsconn/ipfshttp/ipfshttp_test.go +++ b/ipfsconn/ipfshttp/ipfshttp_test.go @@ -294,6 +294,21 @@ func TestRepoStat(t *testing.T) { } } +func TestResolve(t *testing.T) { + ctx := context.Background() + ipfs, mock := testIPFSConnector(t) + defer mock.Close() + defer ipfs.Shutdown(ctx) + + s, err := ipfs.Resolve(ctx, test.TestPathIPFS2) + if err != nil { + t.Error(err) + } + if s.String() != test.TestCidResolved { + t.Errorf("expected different cid, expected: %s, found: %s\n", test.TestCidResolved, s.String()) + } +} + func TestConfigKey(t *testing.T) { ctx := context.Background() ipfs, mock := testIPFSConnector(t) diff --git a/rpc_api.go b/rpc_api.go index bf9fc635..fd6812a4 100644 --- a/rpc_api.go +++ b/rpc_api.go @@ -43,6 +43,20 @@ func (rpcapi *RPCAPI) Unpin(ctx context.Context, in api.PinSerial, out *struct{} return rpcapi.c.Unpin(ctx, c) } +// PinPath resolves path into a cid and runs Cluster.Pin(). +func (rpcapi *RPCAPI) PinPath(ctx context.Context, in api.PinPath, out *api.PinSerial) error { + pin, err := rpcapi.c.PinPath(ctx, in) + *out = pin.ToSerial() + return err +} + +// UnpinPath resolves path into a cid and runs Cluster.Unpin(). +func (rpcapi *RPCAPI) UnpinPath(ctx context.Context, in string, out *api.PinSerial) error { + pin, err := rpcapi.c.UnpinPath(ctx, in) + *out = pin.ToSerial() + return err +} + // Pins runs Cluster.Pins(). func (rpcapi *RPCAPI) Pins(ctx context.Context, in struct{}, out *[]api.PinSerial) error { cidList := rpcapi.c.Pins(ctx) diff --git a/sharness/t0030-ctl-pin.sh b/sharness/t0030-ctl-pin.sh index cc66880c..c813293d 100755 --- a/sharness/t0030-ctl-pin.sh +++ b/sharness/t0030-ctl-pin.sh @@ -49,6 +49,35 @@ test_expect_success IPFS,CLUSTER "wait for data to unpin from cluster with ctl w ipfs-cluster-ctl status "$cid" | grep -q -i "UNPINNED" ' +cid=(`docker exec ipfs sh -c "mkdir -p test1/test2 && touch test1/test2/test3.txt && ipfs add -qr test1"`) + +test_expect_success IPFS,CLUSTER "pin data to cluster with ctl using ipfs paths" ' + ipfs-cluster-ctl pin add "/ipfs/${cid[2]}/test2/test3.txt" && + ipfs-cluster-ctl pin ls "${cid[0]}" | grep -q "${cid[0]}" && + ipfs-cluster-ctl status "${cid[0]}" | grep -q -i "PINNED" +' + +test_expect_success IPFS,CLUSTER "unpin data to cluster with ctl using ipfs paths" ' + removed=(`ipfs-cluster-ctl pin rm "/ipfs/${cid[2]}/test2/test3.txt"`) && + echo "${removed[0]}" | grep -q "${cid[0]}" && + !(ipfs-cluster-ctl pin ls "${cid[0]}" | grep -q "${cid[0]}") && + ipfs-cluster-ctl status "${cid[0]}" | grep -q -i "UNPINNED" +' + +test_expect_success IPFS,CLUSTER "pin data to cluster with ctl using ipns paths" ' + name=`docker exec ipfs sh -c "ipfs name publish -Q ${cid[0]}"` + ipfs-cluster-ctl pin add "/ipns/$name" && + ipfs-cluster-ctl pin ls "${cid[0]}" | grep -q "${cid[0]}" && + ipfs-cluster-ctl status "${cid[0]}" | grep -q -i "PINNED" +' + +test_expect_success IPFS,CLUSTER "unpin data to cluster with ctl using ipns paths" ' + removed=(`ipfs-cluster-ctl pin rm "/ipns/$name"`) && + echo "${removed[0]}" | grep -q "${cid[0]}" && + !(ipfs-cluster-ctl pin ls "${cid[0]}" | grep -q "${cid[0]}") && + ipfs-cluster-ctl status "${cid[0]}" | grep -q -i "UNPINNED" +' + test_clean_ipfs test_clean_cluster diff --git a/test/cids.go b/test/cids.go index 55e43285..46a7c520 100644 --- a/test/cids.go +++ b/test/cids.go @@ -7,12 +7,13 @@ import ( // Common variables used all around tests. var ( - TestCid1 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq" - TestCid2 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmma" - TestCid3 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmb" - TestCid4 = "zb2rhiKhUepkTMw7oFfBUnChAN7ABAvg2hXUwmTBtZ6yxuc57" - TestCid4Data = "Cid4Data" // Cid resulting from block put NOT ipfs add - TestSlowCid1 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmd" + TestCid1 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq" + TestCid2 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmma" + TestCid3 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmb" + TestCid4 = "zb2rhiKhUepkTMw7oFfBUnChAN7ABAvg2hXUwmTBtZ6yxuc57" + TestCid4Data = "Cid4Data" // Cid resulting from block put NOT ipfs add + TestSlowCid1 = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmd" + TestCidResolved = "zb2rhiKhUepkTMw7oFfBUnChAN7ABAvg2hXUwmTBtZ6yxuabc" // ErrorCid is meant to be used as a Cid which causes errors. i.e. the // ipfs mock fails when pinning this CID. ErrorCid = "QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmc" @@ -29,6 +30,18 @@ var ( TestPeerName4 = "TestPeer4" TestPeerName5 = "TestPeer5" TestPeerName6 = "TestPeer6" + + TestPathIPFS1 = "/ipfs/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY" + TestPathIPFS2 = "/ipfs/QmbUNM297ZwxB8CfFAznK7H9YMesDoY6Tt5bPgt5MSCB2u/im.gif" + TestPathIPFS3 = "/ipfs/QmbUNM297ZwxB8CfFAznK7H9YMesDoY6Tt5bPgt5MSCB2u/im.gif/" + TestPathIPNS1 = "/ipns/QmbmSAQNnfGcBAB8M8AsSPxd1TY7cpT9hZ398kXAScn2Ka" + TestPathIPNS2 = "/ipns/QmbmSAQNnfGcBAB8M8AsSPxd1TY7cpT9hZ398kXAScn2Ka/" + TestPathIPLD1 = "/ipld/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY" + TestPathIPLD2 = "/ipld/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY/" + + TestInvalidPath1 = "/invalidkeytype/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY/" + TestInvalidPath2 = "/ipfs/invalidhash" + TestInvalidPath3 = "/ipfs/" ) // MustDecodeCid provides a test helper that ignores diff --git a/test/ipfs_mock.go b/test/ipfs_mock.go index e612b156..442b2b50 100644 --- a/test/ipfs_mock.go +++ b/test/ipfs_mock.go @@ -302,6 +302,8 @@ func (m *IpfsMock) handler(w http.ResponseWriter, r *http.Request) { } j, _ := json.Marshal(resp) w.Write(j) + case "resolve": + w.Write([]byte("{\"Path\":\"" + "/ipfs/" + TestCidResolved + "\"}")) case "config/show": resp := mockConfigResp{ Datastore: struct { diff --git a/test/rpc_api_mock.go b/test/rpc_api_mock.go index e54f3bb4..e6efbad0 100644 --- a/test/rpc_api_mock.go +++ b/test/rpc_api_mock.go @@ -6,12 +6,13 @@ import ( "testing" "time" - "github.com/ipfs/ipfs-cluster/api" - cid "github.com/ipfs/go-cid" + gopath "github.com/ipfs/go-path" rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" + + "github.com/ipfs/ipfs-cluster/api" ) // ErrBadCid is returned when using ErrorCid. Operations with that CID always @@ -52,6 +53,24 @@ func (mock *mockService) Unpin(ctx context.Context, in api.PinSerial, out *struc return nil } +func (mock *mockService) PinPath(ctx context.Context, in api.PinPath, out *api.PinSerial) error { + _, err := gopath.ParsePath(in.Path) + if err != nil { + return err + } + *out = api.PinWithOpts(MustDecodeCid(TestCidResolved), in.PinOptions).ToSerial() + return nil +} + +func (mock *mockService) UnpinPath(ctx context.Context, in string, out *api.PinSerial) error { + _, err := gopath.ParsePath(in) + if err != nil { + return err + } + *out = api.PinCid(MustDecodeCid(TestCidResolved)).ToSerial() + return nil +} + func (mock *mockService) Pins(ctx context.Context, in struct{}, out *[]api.PinSerial) error { opts := api.PinOptions{ ReplicationFactorMin: -1,