From 7f60cb318ce389a3d0e5a641879cb2960e0205d5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 10 Oct 2018 16:00:37 +0200 Subject: [PATCH 01/61] Pubsubmon: Gossipsub License: MIT Signed-off-by: Hector Sanjuan --- monitor/pubsubmon/pubsubmon.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitor/pubsubmon/pubsubmon.go b/monitor/pubsubmon/pubsubmon.go index 7a4bcd47..f85bc230 100644 --- a/monitor/pubsubmon/pubsubmon.go +++ b/monitor/pubsubmon/pubsubmon.go @@ -59,7 +59,7 @@ func New(h host.Host, cfg *Config) (*Monitor, error) { mtrs := metrics.NewStore() checker := metrics.NewChecker(mtrs) - pubsub, err := floodsub.NewFloodSub(ctx, h) + pubsub, err := floodsub.NewGossipSub(ctx, h) if err != nil { cancel() return nil, err From fcbfc7f46a00a3bb203e1013f07f86a71cf7a095 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 11 Oct 2018 11:27:42 +0200 Subject: [PATCH 02/61] Pubsubmon: fix test by reducing gossipsub heartbeat License: MIT Signed-off-by: Hector Sanjuan --- monitor/pubsubmon/pubsubmon_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/monitor/pubsubmon/pubsubmon_test.go b/monitor/pubsubmon/pubsubmon_test.go index 0175017e..4190c43e 100644 --- a/monitor/pubsubmon/pubsubmon_test.go +++ b/monitor/pubsubmon/pubsubmon_test.go @@ -3,20 +3,27 @@ package pubsubmon import ( "context" "fmt" + "strconv" "sync" "testing" "time" + floodsub "github.com/libp2p/go-floodsub" libp2p "github.com/libp2p/go-libp2p" peer "github.com/libp2p/go-libp2p-peer" - peerstore "github.com/libp2p/go-libp2p-peerstore" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" ) +func init() { + // GossipSub needs to heartbeat to discover newly connected hosts + // This speeds things up a little. + floodsub.GossipSubHeartbeatInterval = 50 * time.Millisecond +} + type metricFactory struct { l sync.Mutex counter int From a1132b7b74dda518c4f9100a5b034c73649cbf54 Mon Sep 17 00:00:00 2001 From: "Brian L. McMichael" Date: Tue, 16 Oct 2018 17:21:11 -0400 Subject: [PATCH 03/61] Allow ipfs-cluster-ctl to access files In the snap-installed distribution, calls like `ipfs-cluster-ctl add myfile.txt` fail with error `Error: open myfile.txt: permission denied` due to strict confinement to files in the /snap folder. Modifying confinement to `classic` will allow access to files elsewhere on the system. License: MIT Signed-off-by: brianmcmichael --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 0d8e87fa..47784804 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -4,7 +4,7 @@ summary: Collective pinning and composition for IPFS description: | ipfs-cluster allows to replicate content (by pinning) in multiple IPFS nodes. -confinement: strict +confinement: classic apps: service: From 322e87dd59bf305b461c1167b38527b7a8134452 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 16 Oct 2018 15:23:06 +0200 Subject: [PATCH 04/61] Restapi: Add configurable response headers By default, CORS headers allowing GET requests from everywhere are set. This should facilitate the IPFS Web UI integration with the Cluster API. This commit refactors the sendResponse methods in the API, merging them into one as it was difficult to follow the flows that actually send something to the client. All tests now check the presence of the configured headers too, to make sure no route was missed. License: MIT Signed-off-by: Hector Sanjuan --- api/rest/config.go | 21 ++++- api/rest/restapi.go | 187 +++++++++++++++++++++------------------ api/rest/restapi_test.go | 16 ++++ config_test.go | 14 ++- 4 files changed, 148 insertions(+), 90 deletions(-) diff --git a/api/rest/config.go b/api/rest/config.go index 5902d1e6..ed95146c 100644 --- a/api/rest/config.go +++ b/api/rest/config.go @@ -27,6 +27,15 @@ const ( DefaultIdleTimeout = 120 * time.Second ) +// These are the default values for Config. +var ( + DefaultHeaders = map[string][]string{ + "Access-Control-Allow-Headers": []string{"X-Requested-With", "Range"}, + "Access-Control-Allow-Methods": []string{"GET"}, + "Access-Control-Allow-Origin": []string{"*"}, + } +) + // Config is used to intialize the API object and allows to // customize the behaviour of it. It implements the config.ComponentConfig // interface. @@ -71,6 +80,10 @@ type Config struct { // BasicAuthCreds is a map of username-password pairs // which are authorized to use Basic Authentication BasicAuthCreds map[string]string + + // Headers provides customization for the headers returned + // by the API. By default it sets a CORS policy. + Headers map[string][]string } type jsonConfig struct { @@ -87,7 +100,8 @@ type jsonConfig struct { ID string `json:"id,omitempty"` PrivateKey string `json:"private_key,omitempty"` - BasicAuthCreds map[string]string `json:"basic_auth_credentials"` + BasicAuthCreds map[string]string `json:"basic_auth_credentials"` + Headers map[string][]string `json:"headers"` } // ConfigKey returns a human-friendly identifier for this type of @@ -116,6 +130,9 @@ func (cfg *Config) Default() error { // Auth cfg.BasicAuthCreds = nil + // Headers + cfg.Headers = DefaultHeaders + return nil } @@ -177,6 +194,7 @@ func (cfg *Config) LoadJSON(raw []byte) error { // Other options cfg.BasicAuthCreds = jcfg.BasicAuthCreds + cfg.Headers = jcfg.Headers return cfg.Validate() } @@ -295,6 +313,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) { WriteTimeout: cfg.WriteTimeout.String(), IdleTimeout: cfg.IdleTimeout.String(), BasicAuthCreds: cfg.BasicAuthCreds, + Headers: cfg.Headers, } if cfg.ID != "" { diff --git a/api/rest/restapi.go b/api/rest/restapi.go index 25e50d0c..acc303c9 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -55,6 +55,9 @@ var ( ErrHTTPEndpointNotEnabled = errors.New("the HTTP endpoint is not enabled") ) +// Used by sendResponse to set the right status +const autoStatus = -1 + // For making a random sharding ID var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") @@ -479,7 +482,7 @@ func (api *API) idHandler(w http.ResponseWriter, r *http.Request) { struct{}{}, &idSerial) - sendResponse(w, err, idSerial) + api.sendResponse(w, autoStatus, err, idSerial) } func (api *API) versionHandler(w http.ResponseWriter, r *http.Request) { @@ -490,7 +493,7 @@ func (api *API) versionHandler(w http.ResponseWriter, r *http.Request) { struct{}{}, &v) - sendResponse(w, err, v) + api.sendResponse(w, autoStatus, err, v) } func (api *API) graphHandler(w http.ResponseWriter, r *http.Request) { @@ -500,22 +503,24 @@ func (api *API) graphHandler(w http.ResponseWriter, r *http.Request) { "ConnectGraph", struct{}{}, &graph) - sendResponse(w, err, graph) + api.sendResponse(w, autoStatus, err, graph) } func (api *API) addHandler(w http.ResponseWriter, r *http.Request) { reader, err := r.MultipartReader() if err != nil { - sendErrorResponse(w, http.StatusBadRequest, err.Error()) + api.sendResponse(w, http.StatusBadRequest, err, nil) return } params, err := types.AddParamsFromQuery(r.URL.Query()) if err != nil { - sendErrorResponse(w, http.StatusBadRequest, err.Error()) + api.sendResponse(w, http.StatusBadRequest, err, nil) return } + api.setHeaders(w) + // any errors sent as trailer adderutils.AddMultipartHTTPHandler( api.ctx, @@ -537,7 +542,7 @@ func (api *API) peerListHandler(w http.ResponseWriter, r *http.Request) { struct{}{}, &peersSerial) - sendResponse(w, err, peersSerial) + api.sendResponse(w, autoStatus, err, peersSerial) } func (api *API) peerAddHandler(w http.ResponseWriter, r *http.Request) { @@ -547,13 +552,13 @@ func (api *API) peerAddHandler(w http.ResponseWriter, r *http.Request) { var addInfo peerAddBody err := dec.Decode(&addInfo) if err != nil { - sendErrorResponse(w, 400, "error decoding request body") + api.sendResponse(w, http.StatusBadRequest, errors.New("error decoding request body"), nil) return } _, err = peer.IDB58Decode(addInfo.PeerID) if err != nil { - sendErrorResponse(w, 400, "error decoding peer_id") + api.sendResponse(w, http.StatusBadRequest, errors.New("error decoding peer_id"), nil) return } @@ -563,22 +568,22 @@ func (api *API) peerAddHandler(w http.ResponseWriter, r *http.Request) { "PeerAdd", addInfo.PeerID, &ids) - sendResponse(w, err, ids) + api.sendResponse(w, autoStatus, err, ids) } func (api *API) peerRemoveHandler(w http.ResponseWriter, r *http.Request) { - if p := parsePidOrError(w, r); p != "" { + if p := api.parsePidOrError(w, r); p != "" { err := api.rpcClient.Call("", "Cluster", "PeerRemove", p, &struct{}{}) - sendEmptyResponse(w, err) + api.sendResponse(w, autoStatus, err, nil) } } func (api *API) pinHandler(w http.ResponseWriter, r *http.Request) { - if ps := parseCidOrError(w, r); ps.Cid != "" { + if ps := api.parseCidOrError(w, r); ps.Cid != "" { logger.Debugf("rest api pinHandler: %s", ps.Cid) err := api.rpcClient.Call("", @@ -586,20 +591,20 @@ func (api *API) pinHandler(w http.ResponseWriter, r *http.Request) { "Pin", ps, &struct{}{}) - sendAcceptedResponse(w, err) + api.sendResponse(w, http.StatusAccepted, err, nil) logger.Debug("rest api pinHandler done") } } func (api *API) unpinHandler(w http.ResponseWriter, r *http.Request) { - if ps := parseCidOrError(w, r); ps.Cid != "" { + if ps := api.parseCidOrError(w, r); ps.Cid != "" { logger.Debugf("rest api unpinHandler: %s", ps.Cid) err := api.rpcClient.Call("", "Cluster", "Unpin", ps, &struct{}{}) - sendAcceptedResponse(w, err) + api.sendResponse(w, http.StatusAccepted, err, nil) logger.Debug("rest api unpinHandler done") } } @@ -626,11 +631,11 @@ func (api *API) allocationsHandler(w http.ResponseWriter, r *http.Request) { outPins = append(outPins, pinS) } } - sendResponse(w, err, outPins) + api.sendResponse(w, autoStatus, err, outPins) } func (api *API) allocationHandler(w http.ResponseWriter, r *http.Request) { - if ps := parseCidOrError(w, r); ps.Cid != "" { + if ps := api.parseCidOrError(w, r); ps.Cid != "" { var pin types.PinSerial err := api.rpcClient.Call("", "Cluster", @@ -638,10 +643,10 @@ func (api *API) allocationHandler(w http.ResponseWriter, r *http.Request) { ps, &pin) if err != nil { // errors here are 404s - sendErrorResponse(w, 404, err.Error()) + api.sendResponse(w, http.StatusNotFound, err, nil) return } - sendJSONResponse(w, 200, pin) + api.sendResponse(w, autoStatus, nil, pin) } } @@ -656,7 +661,7 @@ func (api *API) statusAllHandler(w http.ResponseWriter, r *http.Request) { "StatusAllLocal", struct{}{}, &pinInfos) - sendResponse(w, err, pinInfosToGlobal(pinInfos)) + api.sendResponse(w, autoStatus, err, pinInfosToGlobal(pinInfos)) } else { var pinInfos []types.GlobalPinInfoSerial err := api.rpcClient.Call("", @@ -664,7 +669,7 @@ func (api *API) statusAllHandler(w http.ResponseWriter, r *http.Request) { "StatusAll", struct{}{}, &pinInfos) - sendResponse(w, err, pinInfos) + api.sendResponse(w, autoStatus, err, pinInfos) } } @@ -672,7 +677,7 @@ func (api *API) statusHandler(w http.ResponseWriter, r *http.Request) { queryValues := r.URL.Query() local := queryValues.Get("local") - if ps := parseCidOrError(w, r); ps.Cid != "" { + if ps := api.parseCidOrError(w, r); ps.Cid != "" { if local == "true" { var pinInfo types.PinInfoSerial err := api.rpcClient.Call("", @@ -680,7 +685,7 @@ func (api *API) statusHandler(w http.ResponseWriter, r *http.Request) { "StatusLocal", ps, &pinInfo) - sendResponse(w, err, pinInfoToGlobal(pinInfo)) + api.sendResponse(w, autoStatus, err, pinInfoToGlobal(pinInfo)) } else { var pinInfo types.GlobalPinInfoSerial err := api.rpcClient.Call("", @@ -688,7 +693,7 @@ func (api *API) statusHandler(w http.ResponseWriter, r *http.Request) { "Status", ps, &pinInfo) - sendResponse(w, err, pinInfo) + api.sendResponse(w, autoStatus, err, pinInfo) } } } @@ -704,7 +709,7 @@ func (api *API) syncAllHandler(w http.ResponseWriter, r *http.Request) { "SyncAllLocal", struct{}{}, &pinInfos) - sendResponse(w, err, pinInfosToGlobal(pinInfos)) + api.sendResponse(w, autoStatus, err, pinInfosToGlobal(pinInfos)) } else { var pinInfos []types.GlobalPinInfoSerial err := api.rpcClient.Call("", @@ -712,7 +717,7 @@ func (api *API) syncAllHandler(w http.ResponseWriter, r *http.Request) { "SyncAll", struct{}{}, &pinInfos) - sendResponse(w, err, pinInfos) + api.sendResponse(w, autoStatus, err, pinInfos) } } @@ -720,7 +725,7 @@ func (api *API) syncHandler(w http.ResponseWriter, r *http.Request) { queryValues := r.URL.Query() local := queryValues.Get("local") - if ps := parseCidOrError(w, r); ps.Cid != "" { + if ps := api.parseCidOrError(w, r); ps.Cid != "" { if local == "true" { var pinInfo types.PinInfoSerial err := api.rpcClient.Call("", @@ -728,7 +733,7 @@ func (api *API) syncHandler(w http.ResponseWriter, r *http.Request) { "SyncLocal", ps, &pinInfo) - sendResponse(w, err, pinInfoToGlobal(pinInfo)) + api.sendResponse(w, autoStatus, err, pinInfoToGlobal(pinInfo)) } else { var pinInfo types.GlobalPinInfoSerial err := api.rpcClient.Call("", @@ -736,7 +741,7 @@ func (api *API) syncHandler(w http.ResponseWriter, r *http.Request) { "Sync", ps, &pinInfo) - sendResponse(w, err, pinInfo) + api.sendResponse(w, autoStatus, err, pinInfo) } } } @@ -751,9 +756,9 @@ func (api *API) recoverAllHandler(w http.ResponseWriter, r *http.Request) { "RecoverAllLocal", struct{}{}, &pinInfos) - sendResponse(w, err, pinInfosToGlobal(pinInfos)) + api.sendResponse(w, autoStatus, err, pinInfosToGlobal(pinInfos)) } else { - sendErrorResponse(w, 400, "only requests with parameter local=true are supported") + api.sendResponse(w, http.StatusBadRequest, errors.New("only requests with parameter local=true are supported"), nil) } } @@ -761,7 +766,7 @@ func (api *API) recoverHandler(w http.ResponseWriter, r *http.Request) { queryValues := r.URL.Query() local := queryValues.Get("local") - if ps := parseCidOrError(w, r); ps.Cid != "" { + if ps := api.parseCidOrError(w, r); ps.Cid != "" { if local == "true" { var pinInfo types.PinInfoSerial err := api.rpcClient.Call("", @@ -769,7 +774,7 @@ func (api *API) recoverHandler(w http.ResponseWriter, r *http.Request) { "RecoverLocal", ps, &pinInfo) - sendResponse(w, err, pinInfoToGlobal(pinInfo)) + api.sendResponse(w, autoStatus, err, pinInfoToGlobal(pinInfo)) } else { var pinInfo types.GlobalPinInfoSerial err := api.rpcClient.Call("", @@ -777,18 +782,18 @@ func (api *API) recoverHandler(w http.ResponseWriter, r *http.Request) { "Recover", ps, &pinInfo) - sendResponse(w, err, pinInfo) + api.sendResponse(w, autoStatus, err, pinInfo) } } } -func parseCidOrError(w http.ResponseWriter, r *http.Request) types.PinSerial { +func (api *API) parseCidOrError(w http.ResponseWriter, r *http.Request) types.PinSerial { vars := mux.Vars(r) hash := vars["hash"] _, err := cid.Decode(hash) if err != nil { - sendErrorResponse(w, 400, "error decoding Cid: "+err.Error()) + api.sendResponse(w, http.StatusBadRequest, errors.New("error decoding Cid: "+err.Error()), nil) return types.PinSerial{Cid: ""} } @@ -827,12 +832,12 @@ func parseCidOrError(w http.ResponseWriter, r *http.Request) types.PinSerial { return pin } -func parsePidOrError(w http.ResponseWriter, r *http.Request) peer.ID { +func (api *API) parsePidOrError(w http.ResponseWriter, r *http.Request) peer.ID { vars := mux.Vars(r) idStr := vars["peer"] pid, err := peer.IDB58Decode(idStr) if err != nil { - sendErrorResponse(w, 400, "error decoding Peer ID: "+err.Error()) + api.sendResponse(w, http.StatusBadRequest, errors.New("error decoding Peer ID: "+err.Error()), nil) return "" } return pid @@ -855,64 +860,70 @@ func pinInfosToGlobal(pInfos []types.PinInfoSerial) []types.GlobalPinInfoSerial return gPInfos } -func sendResponse(w http.ResponseWriter, err error, resp interface{}) { - if checkErr(w, err) { - sendJSONResponse(w, 200, resp) - } -} +// sendResponse wraps all the logic for writing the response to a request: +// * Write configured headers +// * Write application/json content type +// * Write status: determined automatically if given 0 +// * Write an error if there is or write the response if there is +func (api *API) sendResponse( + w http.ResponseWriter, + status int, + err error, + resp interface{}, +) { -// checkErr takes care of returning standard error responses if we -// pass an error to it. It returns true when everythings OK (no error -// was handled), or false otherwise. -func checkErr(w http.ResponseWriter, err error) bool { + api.setHeaders(w) + enc := json.NewEncoder(w) + + // Send an error if err != nil { - sendErrorResponse(w, http.StatusInternalServerError, err.Error()) - return false - } - return true -} + if status == autoStatus || status < 400 { // set a default error status + status = http.StatusInternalServerError + } + w.WriteHeader(status) -func sendEmptyResponse(w http.ResponseWriter, err error) { - if checkErr(w, err) { - w.WriteHeader(http.StatusNoContent) - } -} + errorResp := types.Error{ + Code: status, + Message: err.Error(), + } + logger.Errorf("sending error response: %d: %s", status, err.Error()) -func sendAcceptedResponse(w http.ResponseWriter, err error) { - if checkErr(w, err) { - w.WriteHeader(http.StatusAccepted) - } -} - -func sendJSONResponse(w http.ResponseWriter, code int, resp interface{}) { - w.Header().Add("Content-Type", "application/json") - w.WriteHeader(code) - if err := json.NewEncoder(w).Encode(resp); err != nil { - logger.Error(err) - } -} - -func sendErrorResponse(w http.ResponseWriter, code int, msg string) { - errorResp := types.Error{ - Code: code, - Message: msg, - } - logger.Errorf("sending error response: %d: %s", code, msg) - sendJSONResponse(w, code, errorResp) -} - -func sendStreamResponse(w http.ResponseWriter, err error, resp <-chan interface{}) { - if !checkErr(w, err) { + if err := enc.Encode(errorResp); err != nil { + logger.Error(err) + } return } - enc := json.NewEncoder(w) - w.Header().Add("Content-Type", "application/octet-stream") - w.WriteHeader(http.StatusOK) - for v := range resp { - err := enc.Encode(v) - if err != nil { + // Send a body + if resp != nil { + if status == autoStatus { + status = http.StatusOK + } + + w.WriteHeader(status) + + if err = enc.Encode(resp); err != nil { logger.Error(err) } + return } + + // Empty response + if status == autoStatus { + status = http.StatusNoContent + } + + w.WriteHeader(status) +} + +// this sets all the headers that are common to all responses +// from this API. Called from sendResponse() and /add. +func (api *API) setHeaders(w http.ResponseWriter) { + for header, values := range api.config.Headers { + for _, val := range values { + w.Header().Add(header, val) + } + } + + w.Header().Add("Content-Type", "application/json") } diff --git a/api/rest/restapi_test.go b/api/rest/restapi_test.go index 849543a2..7de0a3fa 100644 --- a/api/rest/restapi_test.go +++ b/api/rest/restapi_test.go @@ -124,6 +124,17 @@ func processStreamingResp(t *testing.T, httpResp *http.Response, err error, resp } } +func checkHeaders(t *testing.T, rest *API, url string, headers http.Header) { + for k, v := range rest.config.Headers { + if strings.Join(v, ",") != strings.Join(headers[k], ",") { + t.Errorf("%s does not show configured headers: %s", url, k) + } + } + if headers.Get("Content-Type") != "application/json" { + t.Errorf("%s is not application/json", url) + } +} + // makes a libp2p host that knows how to talk to the rest API host. func makeHost(t *testing.T, rest *API) host.Host { h, err := libp2p.New(context.Background()) @@ -185,6 +196,7 @@ func makeGet(t *testing.T, rest *API, url string, resp interface{}) { c := httpClient(t, h, isHTTPS(url)) httpResp, err := c.Get(url) processResp(t, httpResp, err, resp) + checkHeaders(t, rest, url, httpResp.Header) } func makePost(t *testing.T, rest *API, url string, body []byte, resp interface{}) { @@ -193,6 +205,7 @@ func makePost(t *testing.T, rest *API, url string, body []byte, resp interface{} c := httpClient(t, h, isHTTPS(url)) httpResp, err := c.Post(url, "application/json", bytes.NewReader(body)) processResp(t, httpResp, err, resp) + checkHeaders(t, rest, url, httpResp.Header) } func makeDelete(t *testing.T, rest *API, url string, resp interface{}) { @@ -202,6 +215,7 @@ func makeDelete(t *testing.T, rest *API, url string, resp interface{}) { req, _ := http.NewRequest("DELETE", url, bytes.NewReader([]byte{})) httpResp, err := c.Do(req) processResp(t, httpResp, err, resp) + checkHeaders(t, rest, url, httpResp.Header) } func makeStreamingPost(t *testing.T, rest *API, url string, body io.Reader, contentType string, resp interface{}) { @@ -210,6 +224,7 @@ func makeStreamingPost(t *testing.T, rest *API, url string, body io.Reader, cont c := httpClient(t, h, isHTTPS(url)) httpResp, err := c.Post(url, contentType, body) processStreamingResp(t, httpResp, err, resp) + checkHeaders(t, rest, url, httpResp.Header) } type testF func(t *testing.T, url urlF) @@ -251,6 +266,7 @@ func TestRestAPIIDEndpoint(t *testing.T) { rest := testAPI(t) httpsrest := testHTTPSAPI(t) defer rest.Shutdown() + defer httpsrest.Shutdown() tf := func(t *testing.T, url urlF) { id := api.IDSerial{} diff --git a/config_test.go b/config_test.go index 8be40c14..af5dc5b6 100644 --- a/config_test.go +++ b/config_test.go @@ -50,7 +50,19 @@ var testingAPICfg = []byte(`{ "read_timeout": "0", "read_header_timeout": "5s", "write_timeout": "0", - "idle_timeout": "2m0s" + "idle_timeout": "2m0s", + "headers": { + "Access-Control-Allow-Headers": [ + "X-Requested-With", + "Range" + ], + "Access-Control-Allow-Methods": [ + "GET" + ], + "Access-Control-Allow-Origin": [ + "*" + ] + } }`) var testingIpfsCfg = []byte(`{ From 58d35f34ee96fbaafbca5b1ea38453064af8dd99 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 13:40:40 +0200 Subject: [PATCH 05/61] Upgrade to libp2p-6.0.19. Update deps. License: MIT Signed-off-by: Hector Sanjuan --- package.json | 55 ++++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/package.json b/package.json index a02c36ee..21f1a02d 100644 --- a/package.json +++ b/package.json @@ -15,13 +15,13 @@ }, { "author": "whyrusleeping", - "hash": "QmUEqyXr97aUbNmQADHYNknjwjjdVpJXEt1UZXmSG81EV4", + "hash": "QmPL3AKtiaQyYpchZceXBZhZ3MSnoGqJvLZrc7fzDTTQdJ", "name": "go-libp2p", - "version": "6.0.12" + "version": "6.0.19" }, { "author": "hsanjuan", - "hash": "QmaR1KVXHKDyysrMUCT1WKbu9fkFs12U7nLDPhfkswonzj", + "hash": "QmX2KiHWtHDYgxnqBpBEZ7XYdXvSQyhnBJtAKoP4UPGdcf", "name": "go-libp2p-raft", "version": "1.2.13" }, @@ -45,15 +45,15 @@ }, { "author": "hsanjuan", - "hash": "QmXyteEWrYHVJFEA8oX9cSfRp6PJ2kiVsmsFqPMi9ue1Ek", + "hash": "QmUTzMrjVm9MGXPUswihAce991DAi4FFGWEvpXoBzGzi6R", "name": "go-libp2p-gorpc", - "version": "1.0.18" + "version": "1.0.20" }, { "author": "libp2p", - "hash": "QmZaQ3K9PRd5sYYoG1xbTGPtd3N7TYiKBRmcBUTsx8HVET", + "hash": "QmY4Q5JC4vxLEi8EpVxJM4rcRryEVtH1zRKVTAm6BKV1pg", "name": "go-libp2p-pnet", - "version": "3.0.2" + "version": "3.0.4" }, { "author": "ZenGround0", @@ -63,27 +63,27 @@ }, { "author": "dignifiedquire", - "hash": "QmZzgxSj8QpR58KmdeNj97eD66X6xeDAFNjpP2xTY9oKeQ", + "hash": "Qmc4w3gm2TqoEbTYjpPs5FXP8DEB6cuvZWPy6bUTKiht7a", "name": "go-fs-lock", - "version": "0.1.7" + "version": "0.1.8" }, { "author": "hsanjuan", - "hash": "Qmc6tqtdKn1fVCGmU2StfULdXb8xPxmGh19NsYsgVkqjbw", + "hash": "QmXnXKqS94RQipcyMjfjBmfiREdwZgeNGCsUmsoKvM61aq", "name": "go-libp2p-http", - "version": "1.1.1" + "version": "1.1.5" }, { "author": "ipfs", - "hash": "QmR8y7XSkmWSpae9vm7YRES6Bz93pTXX1abeSVKDuNEFeq", + "hash": "QmSaKYjRByLPaC2pnt3ScMKFFgLniZRg11uhaJm8nMJqC8", "name": "go-ipfs-api", - "version": "1.3.6" + "version": "1.4.1" }, { "author": "whyrusleeping", - "hash": "QmY1L5krVk8dv8d74uESmJTXGpoigVYqBVxXXz1aS8aFSb", + "hash": "QmPZFUt4riSxc7PMLX3MGJ5o3uNufJmkTGXEx7SnUaFeP9", "name": "go-libp2p-floodsub", - "version": "0.9.28" + "version": "100.9.36" }, { "author": "whyrusleeping", @@ -99,9 +99,9 @@ }, { "author": "hsanjuan", - "hash": "QmdSeG9s4EQ9TGruJJS9Us38TQDZtMmFGwzTYUDVqNTURm", + "hash": "QmTUTG9Jg9ZRA1EzTPGTDvnwfcfKhDMnqANnP9fe4rSjMR", "name": "go-ipfs-chunker", - "version": "0.1.0" + "version": "0.1.3" }, { "author": "hector", @@ -117,25 +117,20 @@ }, { "author": "why", - "hash": "QmPL8bYtbACcSFFiSr4s2du7Na382NxRADR8hC7D9FkEA2", + "hash": "QmZMVjyWaNCo3Ec44Ce2BcboNuGiQZiJdwW2D3pzwsJdRd", "name": "go-unixfs", - "version": "1.1.1" + "version": "1.1.10" }, { "author": "why", - "hash": "QmXv5mwmQ74r4aiHcNeQ4GAmfB3aWJuqaE4WyDfDfvkgLM", + "hash": "QmVvNkTCx8V9Zei8xuTYTBdUXmbnDRS4iNuw1SztYyhQwQ", "name": "go-merkledag", - "version": "1.1.1" + "version": "1.1.9" }, { - "hash": "QmaXYSwxqJsX3EoGb1ZV2toZ9fXc8hWJPaBW1XAp1h2Tsp", + "hash": "QmSteomMgXnSQxLEY5UpxmkYAd8QF9JuLLeLYBokTHxFru", "name": "go-libp2p-kad-dht", - "version": "4.4.0" - }, - { - "hash": "Qmbq7kGxgcpALGLPaWDyTa6KUq5kBUKdEvkvPZcBkJoLex", - "name": "go-log", - "version": "1.5.6" + "version": "4.4.8" }, { "hash": "QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky", @@ -144,9 +139,9 @@ }, { "author": "hsanjuan", - "hash": "QmRkrpnhZqDxTxwGCsDbuZMr7uCFZHH6SGfrcjgEQwxF3t", + "hash": "QmP2sH3hdhseUsoNBhAZSCkeUvTGgYXxvZjSB2s7yg4aoD", "name": "go-mfs", - "version": "0.1.1" + "version": "0.1.10" }, { "author": "blang", From 562ad713fc8ff739e8c89d74ec9c8484c9edd4eb Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 13:43:57 +0200 Subject: [PATCH 06/61] Update docs for sendResponse License: MIT Signed-off-by: Hector Sanjuan --- api/rest/restapi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/rest/restapi.go b/api/rest/restapi.go index acc303c9..c2bee54a 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -863,7 +863,7 @@ func pinInfosToGlobal(pInfos []types.PinInfoSerial) []types.GlobalPinInfoSerial // sendResponse wraps all the logic for writing the response to a request: // * Write configured headers // * Write application/json content type -// * Write status: determined automatically if given 0 +// * Write status: determined automatically if given "autoStatus" // * Write an error if there is or write the response if there is func (api *API) sendResponse( w http.ResponseWriter, From 86b74c059aa769b524aa19600a53aed35590d982 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 13:47:23 +0200 Subject: [PATCH 07/61] Fix using new libp2p-http License: MIT Signed-off-by: Hector Sanjuan --- api/rest/restapi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/rest/restapi.go b/api/rest/restapi.go index 25e50d0c..f21c255c 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -193,7 +193,7 @@ func (api *API) setupLibp2p(ctx context.Context) error { return nil } - l, err := gostream.Listen(api.host, p2phttp.P2PProtocol) + l, err := gostream.Listen(api.host, p2phttp.DefaultP2PProtocol) if err != nil { return err } From 80baf7d00ae00a58c4eb18541a6af9eb912e5133 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 14:08:56 +0200 Subject: [PATCH 08/61] Fix changed peer ID String() output. License: MIT Signed-off-by: Hector Sanjuan --- cmd/ipfs-cluster-ctl/graph_test.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/ipfs-cluster-ctl/graph_test.go b/cmd/ipfs-cluster-ctl/graph_test.go index e33078da..a9f5d61d 100644 --- a/cmd/ipfs-cluster-ctl/graph_test.go +++ b/cmd/ipfs-cluster-ctl/graph_test.go @@ -33,14 +33,14 @@ var simpleIpfs = `digraph cluster { /* The nodes of the connectivity graph */ /* The cluster-service peers */ -C0 [label="" color="blue2"] -C1 [label="" color="blue2"] -C2 [label="" color="blue2"] +C0 [label="" color="blue2"] +C1 [label="" color="blue2"] +C2 [label="" color="blue2"] /* The ipfs peers */ -I0 [label="" color="goldenrod"] -I1 [label="" color="goldenrod"] -I2 [label="" color="goldenrod"] +I0 [label="" color="goldenrod"] +I1 [label="" color="goldenrod"] +I2 [label="" color="goldenrod"] /* Edges representing active connections in the cluster */ /* The connections among cluster-service peers */ @@ -115,17 +115,17 @@ var allIpfs = `digraph cluster { /* The nodes of the connectivity graph */ /* The cluster-service peers */ -C0 [label="" color="blue2"] -C1 [label="" color="blue2"] -C2 [label="" color="blue2"] +C0 [label="" color="blue2"] +C1 [label="" color="blue2"] +C2 [label="" color="blue2"] /* The ipfs peers */ -I0 [label="" color="goldenrod"] -I1 [label="" color="goldenrod"] -I2 [label="" color="goldenrod"] -I3 [label="" color="goldenrod"] -I4 [label="" color="goldenrod"] -I5 [label="" color="goldenrod"] +I0 [label="" color="goldenrod"] +I1 [label="" color="goldenrod"] +I2 [label="" color="goldenrod"] +I3 [label="" color="goldenrod"] +I4 [label="" color="goldenrod"] +I5 [label="" color="goldenrod"] /* Edges representing active connections in the cluster */ /* The connections among cluster-service peers */ From 7544306ae3e5266e066d7787401e406328883df1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 14:25:22 +0200 Subject: [PATCH 09/61] Fix golint in travis License: MIT Signed-off-by: Hector Sanjuan --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index dfa25a8d..1f248295 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ jobs: - go test -v . -tracker stateless - name: "Golint and go vet" script: - - go get -u github.com/golang/lint/golint + - go get -u golang.org/x/lint/golint - make deps - make check - make service From 7d161087510774ec324691669bd6dd51ed2a6d7a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 15:28:03 +0200 Subject: [PATCH 10/61] Start using libp2p/go-libp2p-gorpc License: MIT Signed-off-by: Hector Sanjuan --- adder/adderutils/adderutils.go | 2 +- adder/local/dag_service.go | 2 +- adder/local/dag_service_test.go | 2 +- adder/sharding/dag.go | 2 +- adder/sharding/dag_service.go | 2 +- adder/sharding/dag_service_test.go | 2 +- adder/sharding/shard.go | 2 +- adder/util.go | 2 +- allocator/ascendalloc/ascendalloc.go | 2 +- allocator/descendalloc/descendalloc.go | 2 +- api/rest/client/methods_test.go | 2 +- api/rest/restapi.go | 2 +- cluster.go | 2 +- cluster_test.go | 2 +- consensus/raft/consensus.go | 2 +- informer/disk/disk.go | 2 +- informer/disk/disk_test.go | 2 +- informer/numpin/numpin.go | 2 +- informer/numpin/numpin_test.go | 2 +- ipfscluster.go | 2 +- ipfsconn/ipfshttp/ipfshttp.go | 2 +- monitor/basic/peer_monitor.go | 2 +- monitor/pubsubmon/pubsubmon.go | 2 +- package.json | 4 ++-- pintracker/maptracker/maptracker.go | 2 +- pintracker/maptracker/maptracker_test.go | 2 +- pintracker/pintracker_test.go | 2 +- pintracker/stateless/stateless.go | 2 +- pintracker/stateless/stateless_test.go | 2 +- test/rpc_api_mock.go | 2 +- 30 files changed, 31 insertions(+), 31 deletions(-) diff --git a/adder/adderutils/adderutils.go b/adder/adderutils/adderutils.go index 55450217..189fe245 100644 --- a/adder/adderutils/adderutils.go +++ b/adder/adderutils/adderutils.go @@ -13,9 +13,9 @@ import ( "github.com/ipfs/ipfs-cluster/adder/sharding" "github.com/ipfs/ipfs-cluster/api" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" ) var logger = logging.Logger("adder") diff --git a/adder/local/dag_service.go b/adder/local/dag_service.go index ad1faf74..0a0c9827 100644 --- a/adder/local/dag_service.go +++ b/adder/local/dag_service.go @@ -9,10 +9,10 @@ import ( adder "github.com/ipfs/ipfs-cluster/adder" "github.com/ipfs/ipfs-cluster/api" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/adder/local/dag_service_test.go b/adder/local/dag_service_test.go index beabc9aa..be1420b6 100644 --- a/adder/local/dag_service_test.go +++ b/adder/local/dag_service_test.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" - rpc "github.com/hsanjuan/go-libp2p-gorpc" + rpc "github.com/libp2p/go-libp2p-gorpc" ) type testRPC struct { diff --git a/adder/sharding/dag.go b/adder/sharding/dag.go index e2bfd548..9c628fa5 100644 --- a/adder/sharding/dag.go +++ b/adder/sharding/dag.go @@ -23,12 +23,12 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/rpcutil" - rpc "github.com/hsanjuan/go-libp2p-gorpc" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" dag "github.com/ipfs/go-merkledag" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" mh "github.com/multiformats/go-multihash" ) diff --git a/adder/sharding/dag_service.go b/adder/sharding/dag_service.go index 1808f7fb..d2c3d5ea 100644 --- a/adder/sharding/dag_service.go +++ b/adder/sharding/dag_service.go @@ -14,10 +14,10 @@ import ( "github.com/ipfs/ipfs-cluster/api" humanize "github.com/dustin/go-humanize" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/adder/sharding/dag_service_test.go b/adder/sharding/dag_service_test.go index 2f92dd34..c6ad83bc 100644 --- a/adder/sharding/dag_service_test.go +++ b/adder/sharding/dag_service_test.go @@ -11,9 +11,9 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" ) func init() { diff --git a/adder/sharding/shard.go b/adder/sharding/shard.go index c925aa6a..84cc6058 100644 --- a/adder/sharding/shard.go +++ b/adder/sharding/shard.go @@ -8,8 +8,8 @@ import ( "github.com/ipfs/ipfs-cluster/api" humanize "github.com/dustin/go-humanize" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/adder/util.go b/adder/util.go index 9fe7c4c5..8863a54e 100644 --- a/adder/util.go +++ b/adder/util.go @@ -8,9 +8,9 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/rpcutil" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/allocator/ascendalloc/ascendalloc.go b/allocator/ascendalloc/ascendalloc.go index 4d7a1165..3b4be58a 100644 --- a/allocator/ascendalloc/ascendalloc.go +++ b/allocator/ascendalloc/ascendalloc.go @@ -8,9 +8,9 @@ import ( "github.com/ipfs/ipfs-cluster/allocator/util" "github.com/ipfs/ipfs-cluster/api" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/allocator/descendalloc/descendalloc.go b/allocator/descendalloc/descendalloc.go index bcf1e7e9..f4264b85 100644 --- a/allocator/descendalloc/descendalloc.go +++ b/allocator/descendalloc/descendalloc.go @@ -8,9 +8,9 @@ import ( "github.com/ipfs/ipfs-cluster/allocator/util" "github.com/ipfs/ipfs-cluster/api" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/api/rest/client/methods_test.go b/api/rest/client/methods_test.go index fbbe096f..3e430844 100644 --- a/api/rest/client/methods_test.go +++ b/api/rest/client/methods_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" diff --git a/api/rest/restapi.go b/api/rest/restapi.go index f21c255c..b7c07da2 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -25,12 +25,12 @@ import ( types "github.com/ipfs/ipfs-cluster/api" mux "github.com/gorilla/mux" - rpc "github.com/hsanjuan/go-libp2p-gorpc" gostream "github.com/hsanjuan/go-libp2p-gostream" p2phttp "github.com/hsanjuan/go-libp2p-http" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" libp2p "github.com/libp2p/go-libp2p" + rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" diff --git a/cluster.go b/cluster.go index dc40ec03..4982a3a3 100644 --- a/cluster.go +++ b/cluster.go @@ -16,8 +16,8 @@ import ( "github.com/ipfs/ipfs-cluster/rpcutil" "github.com/ipfs/ipfs-cluster/state" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" dht "github.com/libp2p/go-libp2p-kad-dht" peer "github.com/libp2p/go-libp2p-peer" diff --git a/cluster_test.go b/cluster_test.go index c46f08a3..77c10e65 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -19,8 +19,8 @@ import ( "github.com/ipfs/ipfs-cluster/state/mapstate" "github.com/ipfs/ipfs-cluster/test" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/consensus/raft/consensus.go b/consensus/raft/consensus.go index ee95a96b..2968099b 100644 --- a/consensus/raft/consensus.go +++ b/consensus/raft/consensus.go @@ -13,9 +13,9 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/state" - rpc "github.com/hsanjuan/go-libp2p-gorpc" logging "github.com/ipfs/go-log" consensus "github.com/libp2p/go-libp2p-consensus" + rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" libp2praft "github.com/libp2p/go-libp2p-raft" diff --git a/informer/disk/disk.go b/informer/disk/disk.go index 2d9491bf..67c951ee 100644 --- a/informer/disk/disk.go +++ b/informer/disk/disk.go @@ -5,8 +5,8 @@ package disk import ( "fmt" - rpc "github.com/hsanjuan/go-libp2p-gorpc" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" "github.com/ipfs/ipfs-cluster/api" ) diff --git a/informer/disk/disk_test.go b/informer/disk/disk_test.go index 9bdde2a7..e8a24346 100644 --- a/informer/disk/disk_test.go +++ b/informer/disk/disk_test.go @@ -5,7 +5,7 @@ import ( "errors" "testing" - rpc "github.com/hsanjuan/go-libp2p-gorpc" + rpc "github.com/libp2p/go-libp2p-gorpc" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" diff --git a/informer/numpin/numpin.go b/informer/numpin/numpin.go index 839596c9..0e026b3e 100644 --- a/informer/numpin/numpin.go +++ b/informer/numpin/numpin.go @@ -5,7 +5,7 @@ package numpin import ( "fmt" - rpc "github.com/hsanjuan/go-libp2p-gorpc" + rpc "github.com/libp2p/go-libp2p-gorpc" "github.com/ipfs/ipfs-cluster/api" ) diff --git a/informer/numpin/numpin_test.go b/informer/numpin/numpin_test.go index d5f9852e..406a656c 100644 --- a/informer/numpin/numpin_test.go +++ b/informer/numpin/numpin_test.go @@ -6,7 +6,7 @@ import ( "github.com/ipfs/ipfs-cluster/api" - rpc "github.com/hsanjuan/go-libp2p-gorpc" + rpc "github.com/libp2p/go-libp2p-gorpc" ) type mockService struct{} diff --git a/ipfscluster.go b/ipfscluster.go index 43c1b785..b73a9e49 100644 --- a/ipfscluster.go +++ b/ipfscluster.go @@ -14,8 +14,8 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/state" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/ipfsconn/ipfshttp/ipfshttp.go b/ipfsconn/ipfshttp/ipfshttp.go index d3a29531..7b7d2c5a 100644 --- a/ipfsconn/ipfshttp/ipfshttp.go +++ b/ipfsconn/ipfshttp/ipfshttp.go @@ -23,10 +23,10 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/rpcutil" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs-cmdkit/files" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" diff --git a/monitor/basic/peer_monitor.go b/monitor/basic/peer_monitor.go index 0b1bb36b..988038a1 100644 --- a/monitor/basic/peer_monitor.go +++ b/monitor/basic/peer_monitor.go @@ -14,8 +14,8 @@ import ( "github.com/ipfs/ipfs-cluster/monitor/metrics" "github.com/ipfs/ipfs-cluster/rpcutil" - rpc "github.com/hsanjuan/go-libp2p-gorpc" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/monitor/pubsubmon/pubsubmon.go b/monitor/pubsubmon/pubsubmon.go index 7a4bcd47..8ef7c72a 100644 --- a/monitor/pubsubmon/pubsubmon.go +++ b/monitor/pubsubmon/pubsubmon.go @@ -10,9 +10,9 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/monitor/metrics" - rpc "github.com/hsanjuan/go-libp2p-gorpc" logging "github.com/ipfs/go-log" floodsub "github.com/libp2p/go-floodsub" + rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" msgpack "github.com/multiformats/go-multicodec/msgpack" diff --git a/package.json b/package.json index 21f1a02d..afa46ef3 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,9 @@ }, { "author": "hsanjuan", - "hash": "QmUTzMrjVm9MGXPUswihAce991DAi4FFGWEvpXoBzGzi6R", + "hash": "QmQe9rBNFc8xgKDnHVebPSDFozdfBWyo9ysDS1WkQKo9w3", "name": "go-libp2p-gorpc", - "version": "1.0.20" + "version": "1.0.21" }, { "author": "libp2p", diff --git a/pintracker/maptracker/maptracker.go b/pintracker/maptracker/maptracker.go index 6b276449..875b8ca4 100644 --- a/pintracker/maptracker/maptracker.go +++ b/pintracker/maptracker/maptracker.go @@ -11,9 +11,9 @@ import ( "github.com/ipfs/ipfs-cluster/pintracker/optracker" "github.com/ipfs/ipfs-cluster/pintracker/util" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/pintracker/maptracker/maptracker_test.go b/pintracker/maptracker/maptracker_test.go index 23ee4f46..2b2c5c77 100644 --- a/pintracker/maptracker/maptracker_test.go +++ b/pintracker/maptracker/maptracker_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" "github.com/ipfs/ipfs-cluster/api" diff --git a/pintracker/pintracker_test.go b/pintracker/pintracker_test.go index 4ea26bc4..637c6a8a 100644 --- a/pintracker/pintracker_test.go +++ b/pintracker/pintracker_test.go @@ -15,8 +15,8 @@ import ( "github.com/ipfs/ipfs-cluster/pintracker/stateless" "github.com/ipfs/ipfs-cluster/test" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/pintracker/stateless/stateless.go b/pintracker/stateless/stateless.go index 5610eb13..6fe05258 100644 --- a/pintracker/stateless/stateless.go +++ b/pintracker/stateless/stateless.go @@ -12,9 +12,9 @@ import ( "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/pintracker/optracker" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/pintracker/stateless/stateless_test.go b/pintracker/stateless/stateless_test.go index 687c4257..8f94fb61 100644 --- a/pintracker/stateless/stateless_test.go +++ b/pintracker/stateless/stateless_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" diff --git a/test/rpc_api_mock.go b/test/rpc_api_mock.go index fcd5ef29..0a3e120c 100644 --- a/test/rpc_api_mock.go +++ b/test/rpc_api_mock.go @@ -8,8 +8,8 @@ import ( "github.com/ipfs/ipfs-cluster/api" - rpc "github.com/hsanjuan/go-libp2p-gorpc" cid "github.com/ipfs/go-cid" + rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" ) From 72e19b4fa103942192f9062aaac498974a01431c Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 17 Oct 2018 15:33:44 +0200 Subject: [PATCH 11/61] Use better version for gorpc License: MIT Signed-off-by: Hector Sanjuan --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index afa46ef3..6a4c1a92 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,9 @@ }, { "author": "hsanjuan", - "hash": "QmQe9rBNFc8xgKDnHVebPSDFozdfBWyo9ysDS1WkQKo9w3", + "hash": "QmUXqxmkMciUCUSNGzdSPvem27ysCUE4nGejXoCykC6A8q", "name": "go-libp2p-gorpc", - "version": "1.0.21" + "version": "1.0.22" }, { "author": "libp2p", From 96af1181445ff885d524a05f84c378474780ad12 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 18 Oct 2018 14:09:19 +0200 Subject: [PATCH 12/61] Fix #577: Try downloading gx and gx-go from local ipfs gateway first This modifies the Makefile so that gx and gx-go downloads are first attempted from the local ipfs gateway (127.0.0.1:8080) and then from the offical gateway (ipfs.io). Make and wget doesn't work well dealing with stuff in subfolders, so I have moved deptools-related rules to the deptools folder in its own Makefile. License: MIT Signed-off-by: Hector Sanjuan --- .gitignore | 2 +- Makefile | 52 ++++++++++++----------------------------------- deptools/Makefile | 45 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 40 deletions(-) create mode 100644 deptools/Makefile diff --git a/.gitignore b/.gitignore index 857decfb..0c09fb07 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ tag_annotation coverage.out cmd/ipfs-cluster-service/ipfs-cluster-service cmd/ipfs-cluster-ctl/ipfs-cluster-ctl -deptools +deptools/gx* sharness/lib/sharness sharness/test-results sharness/trash* diff --git a/Makefile b/Makefile index 7baa8c4b..0e556c83 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,7 @@ -gx_version=v0.13.0 -gx-go_version=v1.8.0 - deptools=deptools - -gx=gx_$(gx_version) -gx-go=gx-go_$(gx-go_version) -gx_bin=$(deptools)/$(gx) -gx-go_bin=$(deptools)/$(gx-go) -bin_env=$(shell go env GOHOSTOS)-$(shell go env GOHOSTARCH) sharness = sharness/lib/sharness +gx=$(deptools)/gx +gx-go=$(deptools)/gx-go # For debugging problematic_test = TestClustersReplicationRealloc @@ -21,9 +14,6 @@ clean: rwundo clean_sharness $(MAKE) -C cmd/ipfs-cluster-ctl clean @rm -rf ./test/testingData -gx-clean: clean - @rm -f $(deptools)/* - install: deps $(MAKE) -C cmd/ipfs-cluster-service install $(MAKE) -C cmd/ipfs-cluster-ctl install @@ -42,38 +32,22 @@ service: deps ctl: deps $(MAKE) -C cmd/ipfs-cluster-ctl ipfs-cluster-ctl -$(gx_bin): - @echo "Downloading gx" - mkdir -p ./$(deptools) - rm -f $(deptools)/gx - wget -nc -O $(gx_bin).tgz https://dist.ipfs.io/gx/$(gx_version)/$(gx)_$(bin_env).tar.gz - tar -zxf $(gx_bin).tgz -C $(deptools) --strip-components=1 gx/gx - mv $(deptools)/gx $(gx_bin) - ln -s $(gx) $(deptools)/gx - rm $(gx_bin).tgz +gx-clean: clean + $(MAKE) -C $(deptools) gx-clean -$(gx-go_bin): - @echo "Downloading gx-go" - mkdir -p ./$(deptools) - rm -f $(deptools)/gx-go - wget -nc -O $(gx-go_bin).tgz https://dist.ipfs.io/gx-go/$(gx-go_version)/$(gx-go)_$(bin_env).tar.gz - tar -zxf $(gx-go_bin).tgz -C $(deptools) --strip-components=1 gx-go/gx-go - mv $(deptools)/gx-go $(gx-go_bin) - ln -s $(gx-go) $(deptools)/gx-go - rm $(gx-go_bin).tgz - -gx: $(gx_bin) $(gx-go_bin) +gx: + $(MAKE) -C $(deptools) gx deps: gx - $(gx_bin) install --global - $(gx-go_bin) rewrite + $(gx) install --global + $(gx-go) rewrite # Run this target before building the docker image # and then gx won't attempt to pull all deps # from the network each time docker_deps: gx - $(gx_bin) install --local - $(gx-go_bin) rewrite + $(gx) install --local + $(gx-go) rewrite check: go vet ./... @@ -101,11 +75,11 @@ clean_sharness: @rm -rf sharness/trash\ directory* rw: gx - $(gx-go_bin) rewrite + $(gx-go) rewrite rwundo: gx - $(gx-go_bin) rewrite --undo + $(gx-go) rewrite --undo publish: rwundo - $(gx_bin) publish + $(gx) publish docker: docker build -t cluster-image -f Dockerfile . diff --git a/deptools/Makefile b/deptools/Makefile new file mode 100644 index 00000000..589106b0 --- /dev/null +++ b/deptools/Makefile @@ -0,0 +1,45 @@ +gx_version=v0.13.0 +gx-go_version=v1.8.0 +gateway=https://ipfs.io +local_gateway=http://127.0.0.1:8080 +dist=dist.ipfs.io +dl_cmd=wget -nc + +gx=gx_$(gx_version) +gx-go=gx-go_$(gx-go_version) +bin_env=$(shell go env GOHOSTOS)-$(shell go env GOHOSTARCH) + +gx_tar=$(gx)_$(bin_env).tar.gz +gx-go_tar=$(gx-go)_$(bin_env).tar.gz + +gx_dist_path=/ipns/$(dist)/gx/$(gx_version)/$(gx_tar) +gx-go_dist_path=/ipns/$(dist)/gx-go/$(gx-go_version)/$(gx-go_tar) + +gx_download_local=$(dl_cmd) $(local_gateway)$(gx_dist_path) +gx_download=$(dl_cmd) $(gateway)$(gx_dist_path) + +gx-go_download_local=$(dl_cmd) $(local_gateway)$(gx-go_dist_path) +gx-go_download=$(dl_cmd) $(gateway)$(gx-go_dist_path) + +$(gx): + @echo "Downloading gx" + rm -f gx + $(gx_download_local) || $(gx_download) + tar -zxf $(gx_tar) --strip-components=1 gx/gx + mv gx $(gx) + ln -s $(gx) gx + rm $(gx_tar) + +$(gx-go): + @echo "Downloading gx-go" + rm -f gx-go + $(gx-go_download_local) || $(gx-go_download) + tar -zxf $(gx-go_tar) --strip-components=1 gx-go/gx-go + mv gx-go $(gx-go) + ln -s $(gx-go) gx-go + rm $(gx-go_tar) + +gx: $(gx) $(gx-go) + +gx-clean: + @rm -f gx* From 96944d0ffba5dc84b3b6101a4a0aa12f5dbccb41 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Wed, 26 Sep 2018 23:15:55 +0530 Subject: [PATCH 13/61] Issue #532 `-f init` should clean up state ipfs-cluster-service -f init` should clean up state License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 61fc02cc..f9ff9927 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -208,6 +208,23 @@ configuration. }, }, Action: func(c *cli.Context) error { + if c.GlobalBool("force") { + err := locker.lock() + checkErr("acquiring execution lock", err) + defer locker.tryUnlock() + + if yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { + cfgMgr, cfgs := makeConfigs() + err = cfgMgr.LoadJSONFromFile(configPath) + checkErr("reading configuration", err) + + err = cleanupState(cfgs.consensusCfg) + checkErr("Cleaning up consensus data", err) + logger.Warningf("the %s folder has been rotated. Next start will use an empty state", cfgs.consensusCfg.GetDataFolder()) + } + + } + userSecret, userSecretDefined := userProvidedSecret(c.Bool("custom-secret")) cfgMgr, cfgs := makeConfigs() From d80532d03259d1d8d9880b3395c9f6a6ff5af367 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 27 Sep 2018 22:51:23 +0530 Subject: [PATCH 14/61] Issue #532 `-f init` should clean up state Do cleanup after configuration in service.json is created License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index f9ff9927..ec98f0a2 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -208,23 +208,6 @@ configuration. }, }, Action: func(c *cli.Context) error { - if c.GlobalBool("force") { - err := locker.lock() - checkErr("acquiring execution lock", err) - defer locker.tryUnlock() - - if yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { - cfgMgr, cfgs := makeConfigs() - err = cfgMgr.LoadJSONFromFile(configPath) - checkErr("reading configuration", err) - - err = cleanupState(cfgs.consensusCfg) - checkErr("Cleaning up consensus data", err) - logger.Warningf("the %s folder has been rotated. Next start will use an empty state", cfgs.consensusCfg.GetDataFolder()) - } - - } - userSecret, userSecretDefined := userProvidedSecret(c.Bool("custom-secret")) cfgMgr, cfgs := makeConfigs() @@ -241,6 +224,14 @@ configuration. // Save saveConfig(cfgMgr, c.GlobalBool("force")) + + // Clean up state + if c.GlobalBool("force") { + err = cleanupState(cfgs.consensusCfg) + checkErr("Cleaning up consensus data", err) + logger.Warningf("the %s folder has been rotated. Starting with an empty state", cfgs.consensusCfg.GetDataFolder()) + + } return nil }, }, From d599ef891bee0677e7d32691b0449e057c38ce7d Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Tue, 2 Oct 2018 19:13:17 +0530 Subject: [PATCH 15/61] Issue #532 `-f init` should clean up state Prompt before cleaning up the state License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 3 +-- sharness/lib/test-lib.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index ec98f0a2..076227f0 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -226,11 +226,10 @@ configuration. saveConfig(cfgMgr, c.GlobalBool("force")) // Clean up state - if c.GlobalBool("force") { + if c.GlobalBool("force") && yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { err = cleanupState(cfgs.consensusCfg) checkErr("Cleaning up consensus data", err) logger.Warningf("the %s folder has been rotated. Starting with an empty state", cfgs.consensusCfg.GetDataFolder()) - } return nil }, diff --git a/sharness/lib/test-lib.sh b/sharness/lib/test-lib.sh index cec3c801..0a017e2c 100755 --- a/sharness/lib/test-lib.sh +++ b/sharness/lib/test-lib.sh @@ -71,7 +71,7 @@ test_cluster_init() { echo "cluster init FAIL: ipfs-cluster-ctl not found" exit 1 fi - ipfs-cluster-service -f --config "test-config" init >"$IPFS_OUTPUT" 2>&1 + printf "y" | ipfs-cluster-service -f --config "test-config" init >"$IPFS_OUTPUT" 2>&1 if [ $? -ne 0 ]; then echo "cluster init FAIL: error on ipfs cluster init" exit 1 From dbc5f97d5e2dc2f3aaf360d95dff7914a9596140 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 4 Oct 2018 20:38:27 +0530 Subject: [PATCH 16/61] Issue #532 `-f init` should clean up state Prompt on init and skip the prompt in init -f (and not have a -y flag) This would be in line with other state subcommands which ask by default and -f skips the prompt. License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/configs.go | 8 +----- cmd/ipfs-cluster-service/main.go | 38 +++++++++++++++++++++++------ sharness/lib/test-lib.sh | 2 +- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/cmd/ipfs-cluster-service/configs.go b/cmd/ipfs-cluster-service/configs.go index 0115b909..31af848a 100644 --- a/cmd/ipfs-cluster-service/configs.go +++ b/cmd/ipfs-cluster-service/configs.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "os" "path/filepath" @@ -67,12 +66,7 @@ func makeConfigs() (*config.Manager, *cfgs) { } } -func saveConfig(cfg *config.Manager, force bool) { - if _, err := os.Stat(configPath); err == nil && !force { - err := fmt.Errorf("%s exists. Try running: %s -f init", configPath, programName) - checkErr("", err) - } - +func saveConfig(cfg *config.Manager) { err := os.MkdirAll(filepath.Dir(configPath), 0700) err = cfg.SaveJSON(configPath) checkErr("saving new configuration", err) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 076227f0..99d20582 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -206,6 +206,10 @@ configuration. Name: "custom-secret, s", Usage: "prompt for the cluster secret", }, + cli.BoolFlag{ + Name: "force, f", + Usage: "forcefully proceed (without prompting) with overwriting configuration and cleaning up state", + }, }, Action: func(c *cli.Context) error { userSecret, userSecretDefined := userProvidedSecret(c.Bool("custom-secret")) @@ -213,6 +217,31 @@ configuration. cfgMgr, cfgs := makeConfigs() defer cfgMgr.Shutdown() // wait for saves + var alreadyInitialized bool + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + alreadyInitialized = true + } + + if alreadyInitialized { + // acquire lock for config folder + err := locker.lock() + checkErr("acquiring execution lock", err) + defer locker.tryUnlock() + + if !c.Bool("force") { + if !yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { + return nil + } + } + + err = cfgMgr.LoadJSONFromFile(configPath) + checkErr("reading configuration", err) + + err = cleanupState(cfgs.consensusCfg) + checkErr("Cleaning up consensus data", err) + logger.Warningf("the %s folder has been rotated. Starting with an empty state", cfgs.consensusCfg.GetDataFolder()) + } + // Generate defaults for all registered components err := cfgMgr.Default() checkErr("generating default configuration", err) @@ -223,14 +252,7 @@ configuration. } // Save - saveConfig(cfgMgr, c.GlobalBool("force")) - - // Clean up state - if c.GlobalBool("force") && yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { - err = cleanupState(cfgs.consensusCfg) - checkErr("Cleaning up consensus data", err) - logger.Warningf("the %s folder has been rotated. Starting with an empty state", cfgs.consensusCfg.GetDataFolder()) - } + saveConfig(cfgMgr) return nil }, }, diff --git a/sharness/lib/test-lib.sh b/sharness/lib/test-lib.sh index 0a017e2c..cec3c801 100755 --- a/sharness/lib/test-lib.sh +++ b/sharness/lib/test-lib.sh @@ -71,7 +71,7 @@ test_cluster_init() { echo "cluster init FAIL: ipfs-cluster-ctl not found" exit 1 fi - printf "y" | ipfs-cluster-service -f --config "test-config" init >"$IPFS_OUTPUT" 2>&1 + ipfs-cluster-service -f --config "test-config" init >"$IPFS_OUTPUT" 2>&1 if [ $? -ne 0 ]; then echo "cluster init FAIL: error on ipfs cluster init" exit 1 From 7fd42cff448554b4e518ae7d608fa46148c648b0 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 4 Oct 2018 21:25:46 +0530 Subject: [PATCH 17/61] Issue #532 `-f init` should clean up state Remove force from global flags and use it as a local flag License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 99d20582..0bad4c0f 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -206,10 +206,6 @@ configuration. Name: "custom-secret, s", Usage: "prompt for the cluster secret", }, - cli.BoolFlag{ - Name: "force, f", - Usage: "forcefully proceed (without prompting) with overwriting configuration and cleaning up state", - }, }, Action: func(c *cli.Context) error { userSecret, userSecretDefined := userProvidedSecret(c.Bool("custom-secret")) @@ -372,12 +368,18 @@ snapshot to be loaded as the cluster state when the cluster peer is restarted. If an argument is provided, cluster will treat it as the path of the file to import. If no argument is provided cluster will read json from stdin `, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "force, f", + Usage: "forcefully proceed with replacing peer state with the exported state, without prompting", + }, + }, Action: func(c *cli.Context) error { err := locker.lock() checkErr("acquiring execution lock", err) defer locker.tryUnlock() - if !c.GlobalBool("force") { + if !c.Bool("force") { if !yesNoPrompt("The peer's state will be replaced. Run with -h for details. Continue? [y/n]:") { return nil } @@ -410,12 +412,18 @@ this state from disk. This command renames cluster's data folder to .old., etc for some rotation factor before permanatly deleting the mth data folder (m currently defaults to 5) `, + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "force, f", + Usage: "forcefully proceed with rotating peer state without prompting", + }, + }, Action: func(c *cli.Context) error { err := locker.lock() checkErr("acquiring execution lock", err) defer locker.tryUnlock() - if !c.GlobalBool("force") { + if !c.Bool("force") { if !yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { return nil } From f77bb9dbcc3131a39c8143f0883cb6b3bb4b2a38 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 4 Oct 2018 23:28:03 +0530 Subject: [PATCH 18/61] Issue #532 `-f init` should clean up state Fix sharness tests by using commands with local flags License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- sharness/t0053-service-state-import.sh | 4 ++-- sharness/t0054-service-state-clean.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sharness/t0053-service-state-import.sh b/sharness/t0053-service-state-import.sh index 5902e410..65f9824a 100755 --- a/sharness/t0053-service-state-import.sh +++ b/sharness/t0053-service-state-import.sh @@ -14,13 +14,13 @@ cluster_kill test_expect_success IPFS,CLUSTER "state import fails on incorrect format" ' sleep 5 && echo "not exactly json" > badImportFile && - test_expect_code 1 ipfs-cluster-service -f --config "test-config" state import badImportFile + test_expect_code 1 ipfs-cluster-service --config "test-config" state import -f badImportFile ' test_expect_success IPFS,CLUSTER,IMPORTSTATE "state import succeeds on correct format" ' sleep 5 cid=`docker exec ipfs sh -c "echo test_53 | ipfs add -q"` && - ipfs-cluster-service -f --debug --config "test-config" state import importState && + ipfs-cluster-service --debug --config "test-config" state import -f importState && cluster_start && sleep 5 && ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && diff --git a/sharness/t0054-service-state-clean.sh b/sharness/t0054-service-state-clean.sh index 156eb2bc..0aaf4098 100755 --- a/sharness/t0054-service-state-clean.sh +++ b/sharness/t0054-service-state-clean.sh @@ -14,7 +14,7 @@ test_expect_success IPFS,CLUSTER "state cleanup refreshes state on restart" ' ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] && cluster_kill && sleep 5 && - ipfs-cluster-service -f --debug --config "test-config" state cleanup && + ipfs-cluster-service --debug --config "test-config" state cleanup -f && cluster_start && sleep 5 && [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] ' @@ -25,8 +25,8 @@ test_expect_success IPFS,CLUSTER "export + cleanup + import == noop" ' [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq ". | length")" ] && cluster_kill && sleep 5 && ipfs-cluster-service --debug --config "test-config" state export -f import.json && - ipfs-cluster-service -f --debug --config "test-config" state cleanup && - ipfs-cluster-service -f --debug --config "test-config" state import import.json && + ipfs-cluster-service --debug --config "test-config" state cleanup -f && + ipfs-cluster-service --debug --config "test-config" state import -f import.json && cluster_start && sleep 5 && ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && From 5f52cc60a8193c7b3673fa7185c3b96dc65ed790 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Fri, 5 Oct 2018 08:22:23 +0530 Subject: [PATCH 19/61] Issue #532 `-f init` should clean up state Change to more appropriate prompt message License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index 0bad4c0f..a3709a01 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -225,7 +225,7 @@ configuration. defer locker.tryUnlock() if !c.Bool("force") { - if !yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { + if !yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost.\nConfiguration(service.json) will be overwritten. Continue? [y/n]:") { return nil } } From 687f49d1b7b7aeb490e77031ecf9e57ad2b0f846 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 18 Oct 2018 09:53:47 +0530 Subject: [PATCH 20/61] Issue #532 `-f init` should clean up state Cosmetic changes License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 18 +++++++++--------- cmd/ipfs-cluster-service/state.go | 10 ++++++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index a3709a01..f2efc877 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -30,6 +30,8 @@ const ( defaultLogLevel = "info" ) +const stateCleanupPrompt = "The peer's state will be removed from the load path. Existing pins may be lost.\nConfiguration(service.json) will be overwritten. Continue? [y/n]:" + // We store a commit id here var commit string @@ -224,18 +226,16 @@ configuration. checkErr("acquiring execution lock", err) defer locker.tryUnlock() - if !c.Bool("force") { - if !yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost.\nConfiguration(service.json) will be overwritten. Continue? [y/n]:") { - return nil - } + if !c.Bool("force") && !yesNoPrompt(stateCleanupPrompt) { + return nil } err = cfgMgr.LoadJSONFromFile(configPath) checkErr("reading configuration", err) - err = cleanupState(cfgs.consensusCfg) + warn, err := cleanupState(cfgs.consensusCfg) checkErr("Cleaning up consensus data", err) - logger.Warningf("the %s folder has been rotated. Starting with an empty state", cfgs.consensusCfg.GetDataFolder()) + logger.Warningf(warn) } // Generate defaults for all registered components @@ -371,7 +371,7 @@ import. If no argument is provided cluster will read json from stdin Flags: []cli.Flag{ cli.BoolFlag{ Name: "force, f", - Usage: "forcefully proceed with replacing peer state with the exported state, without prompting", + Usage: "forcefully proceed with replacing the current state with the given one, without prompting", }, }, Action: func(c *cli.Context) error { @@ -433,9 +433,9 @@ the mth data folder (m currently defaults to 5) err = cfgMgr.LoadJSONFromFile(configPath) checkErr("reading configuration", err) - err = cleanupState(cfgs.consensusCfg) + warn, err := cleanupState(cfgs.consensusCfg) checkErr("Cleaning up consensus data", err) - logger.Warningf("the %s folder has been rotated. Next start will use an empty state", cfgs.consensusCfg.GetDataFolder()) + logger.Warningf(warn) return nil }, }, diff --git a/cmd/ipfs-cluster-service/state.go b/cmd/ipfs-cluster-service/state.go index 3e43cfb3..49dca2fe 100644 --- a/cmd/ipfs-cluster-service/state.go +++ b/cmd/ipfs-cluster-service/state.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io" "io/ioutil" @@ -165,6 +166,11 @@ func exportState(state *mapstate.MapState, w io.Writer) error { } // CleanupState cleans the state -func cleanupState(cCfg *raft.Config) error { - return raft.CleanupRaft(cCfg.GetDataFolder(), cCfg.BackupsRotate) +func cleanupState(cCfg *raft.Config) (string, error) { + err := raft.CleanupRaft(cCfg.GetDataFolder(), cCfg.BackupsRotate) + if err != nil { + return "", err + } + + return fmt.Sprintf("the %s folder has been rotated. Next start will use an empty state", cCfg.GetDataFolder()), nil } From c370e86cca2824c9c5c299ec65bd046218157c8d Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Thu, 18 Oct 2018 18:51:11 +0530 Subject: [PATCH 21/61] Issue #532 `-f init` should clean up state Cosmetic changes License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- cmd/ipfs-cluster-service/main.go | 15 ++++++++------- cmd/ipfs-cluster-service/state.go | 9 ++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/ipfs-cluster-service/main.go b/cmd/ipfs-cluster-service/main.go index f2efc877..f5728c30 100644 --- a/cmd/ipfs-cluster-service/main.go +++ b/cmd/ipfs-cluster-service/main.go @@ -30,7 +30,10 @@ const ( defaultLogLevel = "info" ) -const stateCleanupPrompt = "The peer's state will be removed from the load path. Existing pins may be lost.\nConfiguration(service.json) will be overwritten. Continue? [y/n]:" +const ( + stateCleanupPrompt = "The peer's state will be removed from the load path. Existing pins may be lost." + configurationOverwritePrompt = "Configuration(service.json) will be overwritten." +) // We store a commit id here var commit string @@ -226,16 +229,15 @@ configuration. checkErr("acquiring execution lock", err) defer locker.tryUnlock() - if !c.Bool("force") && !yesNoPrompt(stateCleanupPrompt) { + if !c.Bool("force") && !yesNoPrompt(fmt.Sprintf("%s\n%s Continue? [y/n]:", stateCleanupPrompt, configurationOverwritePrompt)) { return nil } err = cfgMgr.LoadJSONFromFile(configPath) checkErr("reading configuration", err) - warn, err := cleanupState(cfgs.consensusCfg) + err = cleanupState(cfgs.consensusCfg) checkErr("Cleaning up consensus data", err) - logger.Warningf(warn) } // Generate defaults for all registered components @@ -424,7 +426,7 @@ the mth data folder (m currently defaults to 5) defer locker.tryUnlock() if !c.Bool("force") { - if !yesNoPrompt("The peer's state will be removed from the load path. Existing pins may be lost. Continue? [y/n]:") { + if !yesNoPrompt(fmt.Sprintf("%s Continue? [y/n]:", stateCleanupPrompt)) { return nil } } @@ -433,9 +435,8 @@ the mth data folder (m currently defaults to 5) err = cfgMgr.LoadJSONFromFile(configPath) checkErr("reading configuration", err) - warn, err := cleanupState(cfgs.consensusCfg) + err = cleanupState(cfgs.consensusCfg) checkErr("Cleaning up consensus data", err) - logger.Warningf(warn) return nil }, }, diff --git a/cmd/ipfs-cluster-service/state.go b/cmd/ipfs-cluster-service/state.go index 49dca2fe..275823bd 100644 --- a/cmd/ipfs-cluster-service/state.go +++ b/cmd/ipfs-cluster-service/state.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "errors" - "fmt" "io" "io/ioutil" @@ -166,11 +165,11 @@ func exportState(state *mapstate.MapState, w io.Writer) error { } // CleanupState cleans the state -func cleanupState(cCfg *raft.Config) (string, error) { +func cleanupState(cCfg *raft.Config) error { err := raft.CleanupRaft(cCfg.GetDataFolder(), cCfg.BackupsRotate) - if err != nil { - return "", err + if err == nil { + logger.Warningf("the %s folder has been rotated. Next start will use an empty state", cCfg.GetDataFolder()) } - return fmt.Sprintf("the %s folder has been rotated. Next start will use an empty state", cCfg.GetDataFolder()), nil + return err } From 4e3553fb1f44b10bed80e1b814e07bef77a89e9e Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 19 Oct 2018 20:19:52 +0200 Subject: [PATCH 22/61] Fix: escape the add query parameters correctly in the client When adding something that has a name with spaces, things fail very badly. License: MIT Signed-off-by: Hector Sanjuan --- api/add.go | 39 ++++++++++++++------------------- api/rest/client/methods_test.go | 4 ++-- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/api/add.go b/api/add.go index f9ba17e5..19f696be 100644 --- a/api/add.go +++ b/api/add.go @@ -158,29 +158,22 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) { // ToQueryString returns a url query string (key=value&key2=value2&...) func (p *AddParams) ToQueryString() string { - fmtStr := "replication-min=%d&replication-max=%d&name=%s&" - fmtStr += "shard=%t&shard-size=%d&recursive=%t&" - fmtStr += "layout=%s&chunker=%s&raw-leaves=%t&hidden=%t&" - fmtStr += "wrap-with-directory=%t&progress=%t&" - fmtStr += "cid-version=%d&hash=%s" - query := fmt.Sprintf( - fmtStr, - p.ReplicationFactorMin, - p.ReplicationFactorMax, - p.Name, - p.Shard, - p.ShardSize, - p.Recursive, - p.Layout, - p.Chunker, - p.RawLeaves, - p.Hidden, - p.Wrap, - p.Progress, - p.CidVersion, - p.HashFun, - ) - return query + query := url.Values{} + query.Set("replication-min", fmt.Sprintf("%d", p.ReplicationFactorMin)) + query.Set("replication-max", fmt.Sprintf("%d", p.ReplicationFactorMax)) + query.Set("name", p.Name) + query.Set("shard", fmt.Sprintf("%t", p.Shard)) + query.Set("shard-size", fmt.Sprintf("%d", p.ShardSize)) + query.Set("recursive", fmt.Sprintf("%t", p.Recursive)) + query.Set("layout", p.Layout) + query.Set("chunker", p.Chunker) + query.Set("raw-leaves", fmt.Sprintf("%t", p.RawLeaves)) + query.Set("hidden", fmt.Sprintf("%t", p.Hidden)) + query.Set("wrap-with-directory", fmt.Sprintf("%t", p.Wrap)) + query.Set("progress", fmt.Sprintf("%t", p.Progress)) + query.Set("cid-version", fmt.Sprintf("%d", p.CidVersion)) + query.Set("hash", p.HashFun) + return query.Encode() } // Equals checks if p equals p2. diff --git a/api/rest/client/methods_test.go b/api/rest/client/methods_test.go index 3e430844..7c7fa073 100644 --- a/api/rest/client/methods_test.go +++ b/api/rest/client/methods_test.go @@ -137,7 +137,7 @@ func TestPin(t *testing.T) { testF := func(t *testing.T, c Client) { ci, _ := cid.Decode(test.TestCid1) - err := c.Pin(ci, 6, 7, "hello") + err := c.Pin(ci, 6, 7, "hello there") if err != nil { t.Fatal(err) } @@ -443,7 +443,7 @@ func TestAddMultiFile(t *testing.T) { PinOptions: types.PinOptions{ ReplicationFactorMin: -1, ReplicationFactorMax: -1, - Name: "test", + Name: "test something", ShardSize: 1024, }, Shard: false, From fafbf7b929d6fef667c9dae116948af36a1657cd Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Wed, 3 Oct 2018 18:57:38 +0530 Subject: [PATCH 23/61] Issue #449 API endpoint for Peer Monitor metrics Opened new endpoint `GET /health/metrics/` which would respond with metrics of type License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- api/rest/restapi.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api/rest/restapi.go b/api/rest/restapi.go index c0a0b4d8..bf9e7d2b 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -385,6 +385,12 @@ func (api *API) routes() []route { "/health/graph", api.graphHandler, }, + { + "PeerMonitorLatestMetrics", + "GET", + "/health/metrics/{name}", + api.metricsHandler, + }, } } @@ -506,6 +512,19 @@ func (api *API) graphHandler(w http.ResponseWriter, r *http.Request) { api.sendResponse(w, autoStatus, err, graph) } +func (api *API) metricsHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + name := vars["name"] + + var metrics []types.Metric + err := api.rpcClient.Call("", + "Cluster", + "PeerMonitorLatestMetrics", + name, + &metrics) + sendResponse(w, err, metrics) +} + func (api *API) addHandler(w http.ResponseWriter, r *http.Request) { reader, err := r.MultipartReader() if err != nil { From 3ac7d6c9ccb3a45b1f45aec5a9f43e71cba5c556 Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Sun, 7 Oct 2018 22:02:46 +0530 Subject: [PATCH 24/61] Issue #449 API endpoint for Peer Monitor metrics Support the new endpoint for later metrics in `rest/api/client` Support the new method created in `rest/api/client` in ipfs-cluster-ctl. i.e. `ipfs-cluster-ctl health metrics ` would show the peers and the last list of metrics logged for each as returned by the Peer Monitor, in a friendly way. License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- api/rest/client/client.go | 4 ++++ api/rest/client/methods.go | 8 ++++++++ cmd/ipfs-cluster-ctl/formatters.go | 22 ++++++++++++++++++++++ cmd/ipfs-cluster-ctl/main.go | 14 ++++++++++++++ 4 files changed, 48 insertions(+) diff --git a/api/rest/client/client.go b/api/rest/client/client.go index ffc6216f..bbe1ee9e 100644 --- a/api/rest/client/client.go +++ b/api/rest/client/client.go @@ -103,6 +103,10 @@ type Client interface { // GetConnectGraph returns an ipfs-cluster connection graph. The // serialized version, strings instead of pids, is returned GetConnectGraph() (api.ConnectGraphSerial, error) + + // PeerMonitorLatestMetrics returns a map with the latest metrics of matching name + // for the current cluster peers. + PeerMonitorLatestMetrics(name string) ([]api.Metric, error) } // Config allows to configure the parameters to connect diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index e14ae67f..ff45d8ab 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -204,6 +204,14 @@ func (c *defaultClient) GetConnectGraph() (api.ConnectGraphSerial, error) { return graphS, err } +// PeerMonitorLatestMetrics returns a map with the latest metrics of matching name +// for the current cluster peers. +func (c *defaultClient) PeerMonitorLatestMetrics(name string) ([]api.Metric, error) { + var metrics []api.Metric + err := c.do("GET", fmt.Sprintf("/health/metrics/%s", name), nil, nil, &metrics) + return metrics, err +} + // WaitFor is a utility function that allows for a caller to wait for a // paticular status for a CID (as defined by StatusFilterParams). // It returns the final status for that CID and an error, if there was. diff --git a/cmd/ipfs-cluster-ctl/formatters.go b/cmd/ipfs-cluster-ctl/formatters.go index 7c6fc194..87cafff6 100644 --- a/cmd/ipfs-cluster-ctl/formatters.go +++ b/cmd/ipfs-cluster-ctl/formatters.go @@ -24,6 +24,9 @@ func jsonFormatObject(resp interface{}) { jsonFormatPrint(resp.(api.AddedOutput)) case api.Version: jsonFormatPrint(resp.(api.Version)) + case api.Metric: + serial := resp.(api.Metric) + textFormatPrintMetric(&serial) case api.Error: jsonFormatPrint(resp.(api.Error)) case []api.ID: @@ -51,6 +54,9 @@ func jsonFormatObject(resp interface{}) { case []api.AddedOutput: serials := resp.([]api.AddedOutput) jsonFormatPrint(serials) + case []api.Metric: + serials := resp.([]api.Metric) + jsonFormatPrint(serials) default: checkErr("", errors.New("unsupported type returned")) } @@ -84,6 +90,9 @@ func textFormatObject(resp interface{}) { case api.Error: serial := resp.(api.Error) textFormatPrintError(&serial) + case api.Metric: + serial := resp.(api.Metric) + textFormatPrintMetric(&serial) case []api.ID: for _, item := range resp.([]api.ID) { textFormatObject(item) @@ -100,6 +109,10 @@ func textFormatObject(resp interface{}) { for _, item := range resp.([]api.AddedOutput) { textFormatObject(item) } + case []api.Metric: + for _, item := range resp.([]api.Metric) { + textFormatObject(item) + } default: checkErr("", errors.New("unsupported type returned")) } @@ -202,6 +215,15 @@ func textFormatPrintAddedOutput(obj *api.AddedOutput) { fmt.Printf("added %s %s\n", obj.Cid, obj.Name) } +func textFormatPrintMetric(obj *api.Metric) { + fmt.Printf("{\n") + fmt.Printf(" Peer: %s\n", (obj.Peer).String()) + fmt.Printf(" Value: %s\n", obj.Value) + fmt.Printf(" Expire: %d\n", obj.Expire) + fmt.Printf(" Expire: %t\n", obj.Valid) + fmt.Printf("}\n") +} + func textFormatPrintError(obj *api.Error) { fmt.Printf("An error occurred:\n") fmt.Printf(" Code: %d\n", obj.Code) diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index 1a92ad3c..58e1643f 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -762,6 +762,20 @@ graph of the connections. Output is a dot file encoding the cluster's connectio return nil }, }, + { + Name: "metrics", + Usage: "Get latest metrics of matching name for the current cluster peers.", + Description: ` +This command would show the peers and the last list of metrics logged for each as +returned by the Peer Monitor, in a friendly way. +`, + ArgsUsage: "Metric name", + Action: func(c *cli.Context) error { + resp, cerr := globalClient.PeerMonitorLatestMetrics(c.Args().First()) + formatResponse(c, resp, cerr) + return nil + }, + }, }, }, { From ec4588a5ce093b22a95762b6fbc5a6f20247cf6d Mon Sep 17 00:00:00 2001 From: Kishan Sagathiya Date: Sun, 21 Oct 2018 12:04:50 +0530 Subject: [PATCH 25/61] Issue #449 API endpoint for Peer Monitor metrics Rename method PeerMonitorLatestMetrics to Metrics Addressing first round of comment as in https://github.com/ipfs/ipfs-cluster/pull/572#pullrequestreview-165367171 License: MIT Signed-off-by: Kishan Mohanbhai Sagathiya --- api/rest/client/client.go | 4 ++-- api/rest/client/methods.go | 6 +++--- api/rest/restapi.go | 6 +++--- cmd/ipfs-cluster-ctl/formatters.go | 7 +------ cmd/ipfs-cluster-ctl/main.go | 8 ++++---- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/api/rest/client/client.go b/api/rest/client/client.go index bbe1ee9e..e9cdded3 100644 --- a/api/rest/client/client.go +++ b/api/rest/client/client.go @@ -104,9 +104,9 @@ type Client interface { // serialized version, strings instead of pids, is returned GetConnectGraph() (api.ConnectGraphSerial, error) - // PeerMonitorLatestMetrics returns a map with the latest metrics of matching name + // Metrics returns a map with the latest metrics of matching name // for the current cluster peers. - PeerMonitorLatestMetrics(name string) ([]api.Metric, error) + Metrics(name string) ([]api.Metric, error) } // Config allows to configure the parameters to connect diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index ff45d8ab..88298574 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -204,11 +204,11 @@ func (c *defaultClient) GetConnectGraph() (api.ConnectGraphSerial, error) { return graphS, err } -// PeerMonitorLatestMetrics returns a map with the latest metrics of matching name +// Metrics returns a map with the latest metrics of matching name // for the current cluster peers. -func (c *defaultClient) PeerMonitorLatestMetrics(name string) ([]api.Metric, error) { +func (c *defaultClient) Metrics(name string) ([]api.Metric, error) { var metrics []api.Metric - err := c.do("GET", fmt.Sprintf("/health/metrics/%s", name), nil, nil, &metrics) + err := c.do("GET", fmt.Sprintf("/monitor/metrics/%s", name), nil, nil, &metrics) return metrics, err } diff --git a/api/rest/restapi.go b/api/rest/restapi.go index bf9e7d2b..12ddeec6 100644 --- a/api/rest/restapi.go +++ b/api/rest/restapi.go @@ -386,9 +386,9 @@ func (api *API) routes() []route { api.graphHandler, }, { - "PeerMonitorLatestMetrics", + "Metrics", "GET", - "/health/metrics/{name}", + "/monitor/metrics/{name}", api.metricsHandler, }, } @@ -522,7 +522,7 @@ func (api *API) metricsHandler(w http.ResponseWriter, r *http.Request) { "PeerMonitorLatestMetrics", name, &metrics) - sendResponse(w, err, metrics) + api.sendResponse(w, autoStatus, err, metrics) } func (api *API) addHandler(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/ipfs-cluster-ctl/formatters.go b/cmd/ipfs-cluster-ctl/formatters.go index 87cafff6..3d149784 100644 --- a/cmd/ipfs-cluster-ctl/formatters.go +++ b/cmd/ipfs-cluster-ctl/formatters.go @@ -216,12 +216,7 @@ func textFormatPrintAddedOutput(obj *api.AddedOutput) { } func textFormatPrintMetric(obj *api.Metric) { - fmt.Printf("{\n") - fmt.Printf(" Peer: %s\n", (obj.Peer).String()) - fmt.Printf(" Value: %s\n", obj.Value) - fmt.Printf(" Expire: %d\n", obj.Expire) - fmt.Printf(" Expire: %t\n", obj.Valid) - fmt.Printf("}\n") + fmt.Printf("%s: %s | Expire : %d\n", obj.Peer.Pretty(), obj.Value, obj.Expire) } func textFormatPrintError(obj *api.Error) { diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index 58e1643f..02ed42c3 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -764,14 +764,14 @@ graph of the connections. Output is a dot file encoding the cluster's connectio }, { Name: "metrics", - Usage: "Get latest metrics of matching name for the current cluster peers.", + Usage: "List latest metrics logged by this peer", Description: ` -This command would show the peers and the last list of metrics logged for each as -returned by the Peer Monitor, in a friendly way. +This commands displays the latest valid metrics of the given type logged +by this peer for all current cluster peers. `, ArgsUsage: "Metric name", Action: func(c *cli.Context) error { - resp, cerr := globalClient.PeerMonitorLatestMetrics(c.Args().First()) + resp, cerr := globalClient.Metrics(c.Args().First()) formatResponse(c, resp, cerr) return nil }, From 79fe5919d26765b1ca3245cadcdc8d663d01630b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 22 Oct 2018 23:45:33 +0200 Subject: [PATCH 26/61] Fix: escape filter parameter query License: MIT Signed-off-by: Hector Sanjuan --- api/rest/client/methods.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index 88298574..0a133adc 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -109,7 +109,8 @@ func (c *defaultClient) Allocations(filter api.PinType) ([]api.Pin, error) { } } - err := c.do("GET", fmt.Sprintf("/allocations?filter=%s", strings.Join(strFilter, ",")), nil, nil, &pins) + f := url.QueryEscape(strings.Join(strFilter, ",")) + err := c.do("GET", fmt.Sprintf("/allocations?filter=%s", f), nil, nil, &pins) result := make([]api.Pin, len(pins)) for i, p := range pins { result[i] = p.ToPin() From e4fddbc4d7bec95c4e3ac68b0d7208a8e6e62578 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 23 Oct 2018 17:28:50 +0200 Subject: [PATCH 27/61] Add a docker-compose.yml example with 2 peers and auto-bootstrap. License: MIT Signed-off-by: Hector Sanjuan --- .gitignore | 2 + docker-compose.yml | 104 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore index 0c09fb07..e29fd3e9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,11 @@ sharness/test-results sharness/trash* vendor/ + raftFolderFromTest* peerstore shardTesting +compose # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..7f12cd51 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,104 @@ +version: '3.4' + +# This is an example docker-compose file for IPFS Cluster +# It runs two Cluster peers (cluster0, cluster1) attached to two +# IPFS daemons (ipfs0, ipfs1). +# +# It expects a "compose" subfolder as follows where it will store configurations +# and states permanently: +# +# compose/ +# |-- cluster0 +# |-- cluster1 +# |-- ipfs0 +# |-- ipfs1 +# +# +# During the first start, default configurations are created for all peers. + +services: + +################################################################################## +################################################################################## +## Cluster PEER 0 ################################################################ +################################################################################## + + ipfs0: + container_name: ipfs0 + image: ipfs/go-ipfs:release + ports: + - "4001:4001" # ipfs swarm + - "4002:4002/udp" # udp swarm +# - "5001:5001" # expose if needed/wanted +# - "8080:8080" # exposes if needed/wanted + volumes: + - ./compose/ipfs0:/data/ipfs + + cluster0: + container_name: cluster0 + image: ipfs/ipfs-cluster:latest + depends_on: + - ipfs0 + environment: + # Change this! Same for all peers! + CLUSTER_SECRET: adc72159e0ce63350c13de6a5e4d3b2f500232a770cb8c3e7829fc514d89d109 + IPFS_API: /dns4/ipfs0/tcp/5001 + ports: + - "127.0.0.1:9094:9094" # API +# - "9096:9096" # Cluster IPFS Proxy endpoint + volumes: + - ./compose/cluster0:/data/ipfs-cluster + +################################################################################## +################################################################################## +## Cluster PEER 1 ################################################################ +################################################################################## + + ipfs1: + container_name: ipfs1 + image: ipfs/go-ipfs:release + ports: + - "4101:4001" # ipfs swarm + - "4102:4002/udp" # udp swarm +# - "5101:5001" # expose if needed/wanted +# - "8180:8080" # exposes if needed/wanted + volumes: + - ./compose/ipfs1:/data/ipfs + + # cluster1 bootstraps to cluster0 if not bootstrapped before + cluster1: + container_name: cluster1 + image: ipfs/ipfs-cluster:latest + depends_on: + - cluster0 + - ipfs1 + environment: + # Change this! Same for all peers! + CLUSTER_SECRET: adc72159e0ce63350c13de6a5e4d3b2f500232a770cb8c3e7829fc514d89d109 + IPFS_API: /dns4/ipfs1/tcp/5001 + ports: + - "127.0.0.1:9194:9094" # API +# - "9196:9096" # Cluster IPFS Proxy endpoint + volumes: + - ./compose/cluster1:/data/ipfs-cluster + entrypoint: + - "/sbin/tini" + - "--" + # Translation: if state folder does not exist, find cluster0 id and bootstrap + # to it. + command: >- + sh -c ' + cmd="daemon --upgrade" + if [ ! -d /data/ipfs-cluster/raft ]; then + while ! ipfs-cluster-ctl --host /dns4/cluster0/tcp/9094 id; do + sleep 1 + done + pid=`ipfs-cluster-ctl --host /dns4/cluster0/tcp/9094 id | grep -o -E "^(\w+)"` + sleep 10 + cmd="daemon --bootstrap /dns4/cluster0/tcp/9096/ipfs/$$pid" + fi + exec /usr/local/bin/entrypoint.sh $$cmd + ' + +# For adding more peers, copy PEER 1 and rename things to ipfs2, cluster2. +# Keep bootstrapping to cluster0. From ce6ed6ae7accc2dfee4156204197eebf128832c6 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 23 Oct 2018 18:23:58 +0200 Subject: [PATCH 28/61] Test docker compose License: MIT Signed-off-by: Hector Sanjuan --- .travis.yml | 3 ++- Makefile | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1f248295..8a42f8a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,9 +44,10 @@ jobs: - make check - make service - make ctl - - name: "Docker build" + - name: "Docker and Compose build" script: - make docker + - make docker-compose - name: "Sharness" script: - sudo apt-get update diff --git a/Makefile b/Makefile index 0e556c83..818efd82 100644 --- a/Makefile +++ b/Makefile @@ -93,6 +93,13 @@ docker: docker exec tmp-make-cluster-test sh -c "ipfs-cluster-service -v" docker kill tmp-make-cluster-test +docker-compose: + docker-compose up -d + sleep 20 + docker exec cluster0 ipfs-cluster-ctl peers ls | grep -o "Sees 1 other peers" | uniq -c | grep 2 + docker exec cluster1 ipfs-cluster-ctl peers ls | grep -o "Sees 1 other peers" | uniq -c | grep 2 + docker-compose down + prcheck: deps check service ctl test .PHONY: all gx deps test test_sharness clean_sharness rw rwundo publish service ctl install clean gx-clean docker From b86f3cedb49b49adb0fe25009e8aab8c37bb2b9f Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 24 Oct 2018 01:28:05 +0200 Subject: [PATCH 29/61] Files package has been extracted from cmdkit Related: https://github.com/ipfs/go-ipfs-cmdkit/pull/31 License: MIT Signed-off-by: Hector Sanjuan --- adder/adder.go | 2 +- adder/ipfsadd/add.go | 2 +- api/rest/client/client.go | 3 +-- api/rest/client/methods.go | 2 +- ipfsconn/ipfshttp/ipfshttp.go | 2 +- package.json | 11 ++++++----- test/sharding.go | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/adder/adder.go b/adder/adder.go index 44f6c6a7..3389de4e 100644 --- a/adder/adder.go +++ b/adder/adder.go @@ -13,7 +13,7 @@ import ( "github.com/ipfs/ipfs-cluster/api" cid "github.com/ipfs/go-cid" - files "github.com/ipfs/go-ipfs-cmdkit/files" + files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" merkledag "github.com/ipfs/go-merkledag" diff --git a/adder/ipfsadd/add.go b/adder/ipfsadd/add.go index 81e0ea8c..9c47c403 100644 --- a/adder/ipfsadd/add.go +++ b/adder/ipfsadd/add.go @@ -11,7 +11,7 @@ import ( cid "github.com/ipfs/go-cid" chunker "github.com/ipfs/go-ipfs-chunker" - files "github.com/ipfs/go-ipfs-cmdkit/files" + files "github.com/ipfs/go-ipfs-files" posinfo "github.com/ipfs/go-ipfs-posinfo" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log" diff --git a/api/rest/client/client.go b/api/rest/client/client.go index e9cdded3..e93feb27 100644 --- a/api/rest/client/client.go +++ b/api/rest/client/client.go @@ -9,12 +9,11 @@ import ( "net/http" "time" - "github.com/ipfs/go-ipfs-cmdkit/files" - "github.com/ipfs/ipfs-cluster/api" cid "github.com/ipfs/go-cid" shell "github.com/ipfs/go-ipfs-api" + files "github.com/ipfs/go-ipfs-files" logging "github.com/ipfs/go-log" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index 0a133adc..2fcba6dc 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -15,7 +15,7 @@ import ( "github.com/ipfs/ipfs-cluster/api" cid "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-cmdkit/files" + files "github.com/ipfs/go-ipfs-files" peer "github.com/libp2p/go-libp2p-peer" ) diff --git a/ipfsconn/ipfshttp/ipfshttp.go b/ipfsconn/ipfshttp/ipfshttp.go index 7b7d2c5a..288a797f 100644 --- a/ipfsconn/ipfshttp/ipfshttp.go +++ b/ipfsconn/ipfshttp/ipfshttp.go @@ -24,7 +24,7 @@ import ( "github.com/ipfs/ipfs-cluster/rpcutil" cid "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipfs-cmdkit/files" + files "github.com/ipfs/go-ipfs-files" logging "github.com/ipfs/go-log" rpc "github.com/libp2p/go-libp2p-gorpc" peer "github.com/libp2p/go-libp2p-peer" diff --git a/package.json b/package.json index 6a4c1a92..e40952bc 100644 --- a/package.json +++ b/package.json @@ -132,11 +132,6 @@ "name": "go-libp2p-kad-dht", "version": "4.4.8" }, - { - "hash": "QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky", - "name": "go-ipfs-cmdkit", - "version": "1.1.3" - }, { "author": "hsanjuan", "hash": "QmP2sH3hdhseUsoNBhAZSCkeUvTGgYXxvZjSB2s7yg4aoD", @@ -148,6 +143,12 @@ "hash": "QmYRGECuvQnRX73fcvPnGbYijBcGN2HbKZQ7jh26qmLiHG", "name": "semver", "version": "3.5.1" + }, + { + "author": "magik6k", + "hash": "QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC", + "name": "go-ipfs-files", + "version": "1.0.1" } ], "gxVersion": "0.11.0", diff --git a/test/sharding.go b/test/sharding.go index b766fd54..b6de9af1 100644 --- a/test/sharding.go +++ b/test/sharding.go @@ -8,7 +8,7 @@ import ( "path/filepath" "testing" - "github.com/ipfs/go-ipfs-cmdkit/files" + files "github.com/ipfs/go-ipfs-files" ) const shardingTestDir = "shardTesting" From 2a62edd32a3f61badd6927e2bc6726dc6f219b23 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 24 Oct 2018 13:12:36 +0200 Subject: [PATCH 30/61] Nitpicks License: MIT Signed-off-by: Hector Sanjuan --- docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7f12cd51..af414614 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,6 @@ version: '3.4' services: -################################################################################## ################################################################################## ## Cluster PEER 0 ################################################################ ################################################################################## @@ -28,7 +27,6 @@ services: image: ipfs/go-ipfs:release ports: - "4001:4001" # ipfs swarm - - "4002:4002/udp" # udp swarm # - "5001:5001" # expose if needed/wanted # - "8080:8080" # exposes if needed/wanted volumes: @@ -49,7 +47,6 @@ services: volumes: - ./compose/cluster0:/data/ipfs-cluster -################################################################################## ################################################################################## ## Cluster PEER 1 ################################################################ ################################################################################## @@ -59,7 +56,6 @@ services: image: ipfs/go-ipfs:release ports: - "4101:4001" # ipfs swarm - - "4102:4002/udp" # udp swarm # - "5101:5001" # expose if needed/wanted # - "8180:8080" # exposes if needed/wanted volumes: From 0bab730030e52e4c75eb292b0acb48920f559341 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 24 Oct 2018 21:20:53 +0200 Subject: [PATCH 31/61] Set CLUSTER_SECRET from shell License: MIT Signed-off-by: Hector Sanjuan --- Makefile | 3 ++- docker-compose.yml | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 818efd82..56156d18 100644 --- a/Makefile +++ b/Makefile @@ -93,8 +93,9 @@ docker: docker exec tmp-make-cluster-test sh -c "ipfs-cluster-service -v" docker kill tmp-make-cluster-test + docker-compose: - docker-compose up -d + CLUSTER_SECRET=$(shell od -vN 32 -An -tx1 /dev/urandom | tr -d ' \n') docker-compose up -d sleep 20 docker exec cluster0 ipfs-cluster-ctl peers ls | grep -o "Sees 1 other peers" | uniq -c | grep 2 docker exec cluster1 ipfs-cluster-ctl peers ls | grep -o "Sees 1 other peers" | uniq -c | grep 2 diff --git a/docker-compose.yml b/docker-compose.yml index af414614..b46762fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,8 +38,7 @@ services: depends_on: - ipfs0 environment: - # Change this! Same for all peers! - CLUSTER_SECRET: adc72159e0ce63350c13de6a5e4d3b2f500232a770cb8c3e7829fc514d89d109 + CLUSTER_SECRET: ${CLUSTER_SECRET} # From shell variable IPFS_API: /dns4/ipfs0/tcp/5001 ports: - "127.0.0.1:9094:9094" # API @@ -69,8 +68,7 @@ services: - cluster0 - ipfs1 environment: - # Change this! Same for all peers! - CLUSTER_SECRET: adc72159e0ce63350c13de6a5e4d3b2f500232a770cb8c3e7829fc514d89d109 + CLUSTER_SECRET: ${CLUSTER_SECRET} # From shell variable IPFS_API: /dns4/ipfs1/tcp/5001 ports: - "127.0.0.1:9194:9094" # API From 64bd1cb92486b8b4f6a706a28c00d0512e0b89b8 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 25 Oct 2018 12:08:37 +0200 Subject: [PATCH 32/61] Upgrade to libp2p 6.0.23 License: MIT Signed-off-by: Hector Sanjuan --- package.json | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index e40952bc..366d7a86 100644 --- a/package.json +++ b/package.json @@ -15,15 +15,15 @@ }, { "author": "whyrusleeping", - "hash": "QmPL3AKtiaQyYpchZceXBZhZ3MSnoGqJvLZrc7fzDTTQdJ", + "hash": "QmUDTcnDp2WssbmiDLC6aYurUeyt7QeRakHUQMxA2mZ5iB", "name": "go-libp2p", - "version": "6.0.19" + "version": "6.0.23" }, { "author": "hsanjuan", - "hash": "QmX2KiHWtHDYgxnqBpBEZ7XYdXvSQyhnBJtAKoP4UPGdcf", + "hash": "Qmc2WbBR7qA8tYfXwEQKgiHWNn3tZS3e2BhaqqsPoghzYP", "name": "go-libp2p-raft", - "version": "1.2.13" + "version": "1.2.14" }, { "author": "urfave", @@ -45,9 +45,9 @@ }, { "author": "hsanjuan", - "hash": "QmUXqxmkMciUCUSNGzdSPvem27ysCUE4nGejXoCykC6A8q", + "hash": "Qmas5ivzqmatd1tJWppjcupYY8YS2DJHJZxSMsMoLbCQ2R", "name": "go-libp2p-gorpc", - "version": "1.0.22" + "version": "1.0.23" }, { "author": "libp2p", @@ -69,21 +69,21 @@ }, { "author": "hsanjuan", - "hash": "QmXnXKqS94RQipcyMjfjBmfiREdwZgeNGCsUmsoKvM61aq", + "hash": "QmbuqT17YGSLRGiDRSJqVoXzTXrj4R5y6u4px2q42iyf23", "name": "go-libp2p-http", - "version": "1.1.5" + "version": "1.1.6" }, { "author": "ipfs", - "hash": "QmSaKYjRByLPaC2pnt3ScMKFFgLniZRg11uhaJm8nMJqC8", + "hash": "QmQcy2Frmpvm5D5oiWoKXfs71eh3MXTv7LN12J39hJioBi", "name": "go-ipfs-api", - "version": "1.4.1" + "version": "1.4.2" }, { "author": "whyrusleeping", - "hash": "QmPZFUt4riSxc7PMLX3MGJ5o3uNufJmkTGXEx7SnUaFeP9", + "hash": "Qmc71UCoZWor2ADQAWjNnrSVgEtPpEsQbCXf9Gs7WWz7MV", "name": "go-libp2p-floodsub", - "version": "100.9.36" + "version": "100.10.2" }, { "author": "whyrusleeping", @@ -117,26 +117,26 @@ }, { "author": "why", - "hash": "QmZMVjyWaNCo3Ec44Ce2BcboNuGiQZiJdwW2D3pzwsJdRd", + "hash": "QmTJUySFxXjh54zEoFbzQEmGD3yj89XKS3A28y7Nqsn1TC", "name": "go-unixfs", - "version": "1.1.10" + "version": "1.1.15" }, { "author": "why", - "hash": "QmVvNkTCx8V9Zei8xuTYTBdUXmbnDRS4iNuw1SztYyhQwQ", + "hash": "QmY8BMUSpCwNiTmFhACmC9Bt1qT63cHP35AoQAus4x14qH", "name": "go-merkledag", - "version": "1.1.9" + "version": "1.1.13" }, { - "hash": "QmSteomMgXnSQxLEY5UpxmkYAd8QF9JuLLeLYBokTHxFru", + "hash": "QmQHnqaNULV8WeUGgh97o9K3KAW6kWQmDyNf9UuikgnPTe", "name": "go-libp2p-kad-dht", - "version": "4.4.8" + "version": "4.4.12" }, { "author": "hsanjuan", - "hash": "QmP2sH3hdhseUsoNBhAZSCkeUvTGgYXxvZjSB2s7yg4aoD", + "hash": "QmcctvPwYdaJvM4VefQnKAfU2ppSVfjpECvBVVEm18qk68", "name": "go-mfs", - "version": "0.1.10" + "version": "0.1.15" }, { "author": "blang", From f52a5fd7028939e6fdfd0024d2eb41bd95b230cd Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 25 Oct 2018 12:40:33 +0200 Subject: [PATCH 33/61] Fix publishing snaps Snaps won't publish with classic confimement or non-edge. For the moment this works around that so that, at least, there is releases on 'edge' with strict confinement. License: MIT Signed-off-by: Hector Sanjuan --- .travis.yml | 5 ++--- snap/snapcraft.yaml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a42f8a1..9671d0ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,17 +61,16 @@ jobs: if: (NOT type IN (pull_request)) AND (fork = false) AND (tag =~ ^v\d+\.\d+\.\d+$) script: - openssl aes-256-cbc -K $encrypted_5a1cb914c6c9_key -iv $encrypted_5a1cb914c6c9_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d - - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt update -qq && cd $(pwd) && ./snap/snap-multiarch.sh stable" + - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt update -qq && cd $(pwd) && ./snap/snap-multiarch.sh edge" # should be stable - stage: "Snapcraft deployment stage (Candidate)" name: "Deploy Snapcraft" if: (NOT type IN (pull_request)) AND (fork = false) AND (tag =~ ^v\d+\.\d+\.\d+-rc\d+$) script: - openssl aes-256-cbc -K $encrypted_5a1cb914c6c9_key -iv $encrypted_5a1cb914c6c9_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d - - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt update -qq && cd $(pwd) && ./snap/snap-multiarch.sh candidate" + - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt update -qq && cd $(pwd) && ./snap/snap-multiarch.sh edge" # should be candidate - stage: "Snapcraft deployment stage (Edge)" name: "Deploy Snapcraft" if: (NOT type IN (pull_request)) AND (branch = master) AND (fork = false) AND (tag IS NOT present) script: - openssl aes-256-cbc -K $encrypted_5a1cb914c6c9_key -iv $encrypted_5a1cb914c6c9_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt update -qq && cd $(pwd) && ./snap/snap-multiarch.sh edge" - diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 47784804..0d8e87fa 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -4,7 +4,7 @@ summary: Collective pinning and composition for IPFS description: | ipfs-cluster allows to replicate content (by pinning) in multiple IPFS nodes. -confinement: classic +confinement: strict apps: service: From 9330ac82e23b850d6d75e1e09338736073e08456 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 25 Oct 2018 14:55:01 +0200 Subject: [PATCH 34/61] Fix tests with latest libp2p License: MIT Signed-off-by: Hector Sanjuan --- api/rest/client/client_test.go | 12 ++++++++---- consensus/raft/consensus_test.go | 14 ++------------ ipfscluster_test.go | 2 +- peer_manager_test.go | 10 ++++++++-- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/api/rest/client/client_test.go b/api/rest/client/client_test.go index 911003b6..8daac083 100644 --- a/api/rest/client/client_test.go +++ b/api/rest/client/client_test.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/ipfs-cluster/test" libp2p "github.com/libp2p/go-libp2p" + peer "github.com/libp2p/go-libp2p-peer" pnet "github.com/libp2p/go-libp2p-pnet" ma "github.com/multiformats/go-multiaddr" ) @@ -59,9 +60,13 @@ func apiMAddr(a *rest.API) ma.Multiaddr { } func peerMAddr(a *rest.API) ma.Multiaddr { - listenAddr := a.Host().Addrs()[0] - ipfsAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s", a.Host().ID().Pretty())) - return listenAddr.Encapsulate(ipfsAddr) + ipfsAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s", peer.IDB58Encode(a.Host().ID()))) + for _, a := range a.Host().Addrs() { + if _, err := a.ValueForProtocol(ma.P_IP4); err == nil { + return a.Encapsulate(ipfsAddr) + } + } + return nil } func testClientHTTP(t *testing.T, api *rest.API) *defaultClient { @@ -87,7 +92,6 @@ func testClientLibp2p(t *testing.T, api *rest.API) *defaultClient { if err != nil { t.Fatal(err) } - return c.(*defaultClient) } diff --git a/consensus/raft/consensus_test.go b/consensus/raft/consensus_test.go index 6294c55d..b5b85f11 100644 --- a/consensus/raft/consensus_test.go +++ b/consensus/raft/consensus_test.go @@ -15,22 +15,12 @@ import ( libp2p "github.com/libp2p/go-libp2p" host "github.com/libp2p/go-libp2p-host" peerstore "github.com/libp2p/go-libp2p-peerstore" - ma "github.com/multiformats/go-multiaddr" ) func cleanRaft(idn int) { os.RemoveAll(fmt.Sprintf("raftFolderFromTests-%d", idn)) } -func consensusListenAddr(c *Consensus) ma.Multiaddr { - return c.host.Addrs()[0] -} - -func consensusAddr(c *Consensus) ma.Multiaddr { - cAddr, _ := ma.NewMultiaddr(fmt.Sprintf("%s/ipfs/%s", consensusListenAddr(c), c.host.ID().Pretty())) - return cAddr -} - func testPin(c cid.Cid) api.Pin { p := api.PinCid(c) p.ReplicationFactorMin = -1 @@ -174,7 +164,7 @@ func TestConsensusAddPeer(t *testing.T) { defer cc.Shutdown() defer cc2.Shutdown() - cc.host.Peerstore().AddAddr(cc2.host.ID(), consensusListenAddr(cc2), peerstore.PermanentAddrTTL) + cc.host.Peerstore().AddAddrs(cc2.host.ID(), cc2.host.Addrs(), peerstore.PermanentAddrTTL) err := cc.AddPeer(cc2.host.ID()) if err != nil { t.Error("the operation did not make it to the log:", err) @@ -205,7 +195,7 @@ func TestConsensusRmPeer(t *testing.T) { defer cc.Shutdown() defer cc2.Shutdown() - cc.host.Peerstore().AddAddr(cc2.host.ID(), consensusListenAddr(cc2), peerstore.PermanentAddrTTL) + cc.host.Peerstore().AddAddrs(cc2.host.ID(), cc2.host.Addrs(), peerstore.PermanentAddrTTL) err := cc.AddPeer(cc2.host.ID()) if err != nil { diff --git a/ipfscluster_test.go b/ipfscluster_test.go index fd6461a3..1a7a7dee 100644 --- a/ipfscluster_test.go +++ b/ipfscluster_test.go @@ -269,7 +269,7 @@ func createClusters(t *testing.T) ([]*Cluster, []*test.IpfsMock) { // Start first node clusters[0] = createCluster(t, hosts[0], cfgs[0], raftCons[0], apis[0], ipfss[0], states[0], trackers[0], mons[0], allocs[0], infs[0]) <-clusters[0].Ready() - bootstrapAddr, _ := ma.NewMultiaddr(fmt.Sprintf("%s/ipfs/%s", clusters[0].host.Addrs()[0], clusters[0].id.Pretty())) + bootstrapAddr := clusterAddr(clusters[0]) // Start the rest and join for i := 1; i < nClusters; i++ { diff --git a/peer_manager_test.go b/peer_manager_test.go index 61db555d..9b6859ca 100644 --- a/peer_manager_test.go +++ b/peer_manager_test.go @@ -50,8 +50,14 @@ func peerManagerClusters(t *testing.T) ([]*Cluster, []*test.IpfsMock) { } func clusterAddr(c *Cluster) ma.Multiaddr { - cAddr, _ := ma.NewMultiaddr(fmt.Sprintf("%s/ipfs/%s", c.host.Addrs()[0], c.id.Pretty())) - return cAddr + for _, a := range c.host.Addrs() { + if _, err := a.ValueForProtocol(ma.P_IP4); err == nil { + p := peer.IDB58Encode(c.id) + cAddr, _ := ma.NewMultiaddr(fmt.Sprintf("%s/ipfs/%s", a, p)) + return cAddr + } + } + return nil } func TestClustersPeerAdd(t *testing.T) { From 3ab3c2afa0cd780ee1644dd11a5fec56221e3ccb Mon Sep 17 00:00:00 2001 From: Adrian Lanzafame Date: Fri, 26 Oct 2018 19:38:30 +1000 Subject: [PATCH 35/61] add env config; refactor LoadJSON tests License: MIT Signed-off-by: Adrian Lanzafame --- cluster_config.go | 19 ++- cluster_config_test.go | 257 +++++++++++++++++++++++++---------------- package.json | 6 + 3 files changed, 178 insertions(+), 104 deletions(-) diff --git a/cluster_config.go b/cluster_config.go index 873d5803..c9268771 100644 --- a/cluster_config.go +++ b/cluster_config.go @@ -11,6 +11,8 @@ import ( "sync" "time" + "github.com/kelseyhightower/envconfig" + "github.com/ipfs/ipfs-cluster/config" crypto "github.com/libp2p/go-libp2p-crypto" @@ -240,8 +242,7 @@ func isReplicationFactorValid(rplMin, rplMax int) error { return errors.New("cluster.replication_factor_max is wrong") } - if (rplMin == -1 && rplMax != -1) || - (rplMin != -1 && rplMax == -1) { + if (rplMin == -1 && rplMax != -1) || (rplMin != -1 && rplMax == -1) { return errors.New("cluster.replication_factor_min and max must be -1 when one of them is") } return nil @@ -272,6 +273,8 @@ func (cfg *Config) setDefaults() { // sets the Config fields from it. Note that it should be JSON // as generated by ToJSON(). func (cfg *Config) LoadJSON(raw []byte) error { + cfg.setDefaults() + jcfg := &configJSON{} err := json.Unmarshal(raw, jcfg) if err != nil { @@ -279,10 +282,6 @@ func (cfg *Config) LoadJSON(raw []byte) error { return err } - // Make sure all non-defined keys have good values. - cfg.setDefaults() - config.SetIfNotDefault(jcfg.PeerstoreFile, &cfg.PeerstoreFile) - if jcfg.Peers != nil || jcfg.Bootstrap != nil { logger.Error(` Your configuration is using cluster.Peers and/or cluster.Bootstrap @@ -303,6 +302,12 @@ for more information.`) return errors.New("cluster.Peers and cluster.Bootstrap keys have been deprecated") } + // override json config with env var + err = envconfig.Process(cfg.ConfigKey(), jcfg) + if err != nil { + return err + } + parseDuration := func(txt string) time.Duration { d, _ := time.ParseDuration(txt) if txt != "" && d == 0 { @@ -311,6 +316,8 @@ for more information.`) return d } + config.SetIfNotDefault(jcfg.PeerstoreFile, &cfg.PeerstoreFile) + id, err := peer.IDB58Decode(jcfg.ID) if err != nil { err = fmt.Errorf("error decoding cluster ID: %s", err) diff --git a/cluster_config_test.go b/cluster_config_test.go index 004be665..2af5ec0b 100644 --- a/cluster_config_test.go +++ b/cluster_config_test.go @@ -2,6 +2,7 @@ package ipfscluster import ( "encoding/json" + "os" "testing" ) @@ -23,118 +24,178 @@ var ccfgTestJSON = []byte(` `) func TestLoadJSON(t *testing.T) { - cfg := &Config{} - err := cfg.LoadJSON(ccfgTestJSON) - if err != nil { - t.Fatal(err) + loadJSON := func(t *testing.T) (*Config, error) { + cfg := &Config{} + err := cfg.LoadJSON(ccfgTestJSON) + if err != nil { + return cfg, err + } + return cfg, nil } - if cfg.Peername != "testpeer" { - t.Error("expected peername 'testpeer'") + t.Run("basic", func(t *testing.T) { + cfg := &Config{} + err := cfg.LoadJSON(ccfgTestJSON) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("peername", func(t *testing.T) { + cfg, err := loadJSON(t) + if err != nil { + t.Error(err) + } + if cfg.Peername != "testpeer" { + t.Error("expected peername 'testpeer'") + } + }) + + t.Run("expected replication factor", func(t *testing.T) { + cfg, err := loadJSON(t) + if err != nil { + t.Error(err) + } + if cfg.ReplicationFactorMin != 5 { + t.Error("expected replication factor min == 5") + } + }) + + t.Run("expected disable_repinning", func(t *testing.T) { + cfg, err := loadJSON(t) + if err != nil { + t.Error(err) + } + if !cfg.DisableRepinning { + t.Error("expected disable_repinning to be true") + } + }) + + loadJSON2 := func(t *testing.T, f func(j *configJSON)) (*Config, error) { + cfg := &Config{} + j := &configJSON{} + json.Unmarshal(ccfgTestJSON, j) + f(j) + tst, err := json.Marshal(j) + if err != nil { + return cfg, err + } + err = cfg.LoadJSON(tst) + if err != nil { + return cfg, err + } + return cfg, nil } - if cfg.ReplicationFactorMin != 5 { - t.Error("expected replication factor min == 5") - } + t.Run("bad id", func(t *testing.T) { + _, err := loadJSON2(t, func(j *configJSON) { j.ID = "abc" }) + if err == nil { + t.Error("expected error decoding ID") + } + }) - if !cfg.DisableRepinning { - t.Error("expected disable_repinning to be true") - } + t.Run("empty default peername", func(t *testing.T) { + cfg, err := loadJSON2(t, func(j *configJSON) { j.Peername = "" }) + if err != nil { + t.Error(err) + } + if cfg.Peername == "" { + t.Error("expected default peername") + } + }) - j := &configJSON{} + t.Run("bad private key", func(t *testing.T) { + _, err := loadJSON2(t, func(j *configJSON) { j.PrivateKey = "abc" }) + if err == nil { + t.Error("expected error parsing private key") + } + }) - json.Unmarshal(ccfgTestJSON, j) - j.ID = "abc" - tst, _ := json.Marshal(j) - err = cfg.LoadJSON(tst) - if err == nil { - t.Error("expected error decoding ID") - } + t.Run("bad listen multiaddress", func(t *testing.T) { + _, err := loadJSON2(t, func(j *configJSON) { j.ListenMultiaddress = "abc" }) + if err == nil { + t.Error("expected error parsing listen_multiaddress") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.Peername = "" - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if cfg.Peername == "" { - t.Error("expected default peername") - } + t.Run("bad secret", func(t *testing.T) { + _, err := loadJSON2(t, func(j *configJSON) { j.Secret = "abc" }) + if err == nil { + t.Error("expected error decoding secret") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.PrivateKey = "abc" - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if err == nil { - t.Error("expected error parsing private key") - } + t.Run("default replication factors", func(t *testing.T) { + cfg, err := loadJSON2( + t, + func(j *configJSON) { + j.ReplicationFactor = 0 + j.ReplicationFactorMin = 0 + j.ReplicationFactorMax = 0 + }, + ) + if err != nil { + t.Error(err) + } + if cfg.ReplicationFactorMin != -1 || cfg.ReplicationFactorMax != -1 { + t.Error("expected default replication factor") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.ListenMultiaddress = "abc" - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if err == nil { - t.Error("expected error parsing listen_multiaddress") - } + t.Run("replication factor min/max override", func(t *testing.T) { + cfg, err := loadJSON2(t, func(j *configJSON) { j.ReplicationFactor = 3 }) + if err != nil { + t.Error(err) + } + if cfg.ReplicationFactorMin != 3 || cfg.ReplicationFactorMax != 3 { + t.Error("expected replicationFactor Min/Max override") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.Secret = "abc" - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if err == nil { - t.Error("expected error decoding secret") - } + t.Run("only replication factor min set to -1", func(t *testing.T) { + _, err := loadJSON2(t, func(j *configJSON) { j.ReplicationFactorMin = -1 }) + if err == nil { + t.Error("expected error when only one replication factor is -1") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.ReplicationFactor = 0 - j.ReplicationFactorMin = 0 - j.ReplicationFactorMax = 0 - tst, _ = json.Marshal(j) - cfg.LoadJSON(tst) - if cfg.ReplicationFactorMin != -1 || cfg.ReplicationFactorMax != -1 { - t.Error("expected default replication factor") - } + t.Run("replication factor min > max", func(t *testing.T) { + _, err := loadJSON2( + t, + func(j *configJSON) { + j.ReplicationFactorMin = 5 + j.ReplicationFactorMax = 4 + }, + ) + if err == nil { + t.Error("expected error when only rplMin > rplMax") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.ReplicationFactor = 3 - tst, _ = json.Marshal(j) - cfg.LoadJSON(tst) - if cfg.ReplicationFactorMin != 3 || cfg.ReplicationFactorMax != 3 { - t.Error("expected replicationFactor Min/Max override") - } + t.Run("default replication factor", func(t *testing.T) { + cfg, err := loadJSON2( + t, + func(j *configJSON) { + j.ReplicationFactorMin = 0 + j.ReplicationFactorMax = 0 + }, + ) + if err != nil { + t.Error(err) + } + if cfg.ReplicationFactorMin != -1 || cfg.ReplicationFactorMax != -1 { + t.Error("expected default replication factors") + } + }) - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.ReplicationFactorMin = -1 - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if err == nil { - t.Error("expected error when only one replication factor is -1") - } - - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.ReplicationFactorMin = 5 - j.ReplicationFactorMax = 4 - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if err == nil { - t.Error("expected error when only rplMin > rplMax") - } - - j = &configJSON{} - json.Unmarshal(ccfgTestJSON, j) - j.ReplicationFactorMin = 0 - j.ReplicationFactorMax = 0 - tst, _ = json.Marshal(j) - err = cfg.LoadJSON(tst) - if cfg.ReplicationFactorMin != -1 || cfg.ReplicationFactorMax != -1 { - t.Error("expected default replication factors") - } + t.Run("env var override", func(t *testing.T) { + os.Setenv("CLUSTER_PEERNAME", "envsetpeername") + cfg := &Config{} + cfg.LoadJSON(ccfgTestJSON) + if cfg.Peername != "envsetpeername" { + t.Fatal("failed to override peername with env var") + } + }) } func TestToJSON(t *testing.T) { diff --git a/package.json b/package.json index 366d7a86..3522d994 100644 --- a/package.json +++ b/package.json @@ -149,6 +149,12 @@ "hash": "QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC", "name": "go-ipfs-files", "version": "1.0.1" + }, + { + "author": "lanzafame", + "hash": "QmVPrFJ6dYJyt7uULgE7wouhvXDzYBZVjYvjMCE5Uqxgfy", + "name": "envconfig", + "version": "0.0.0" } ], "gxVersion": "0.11.0", From 19b1124999cc9879156fdedc60073eb9e21f815f Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 23 Oct 2018 20:21:27 +0200 Subject: [PATCH 36/61] Make metrics human Issue #572 exposes metrics but they carry the peer ID in binary. This was ok with our internal codecs but it doesn't seem to work very well with json, and makes the output format unusable. This makes the Metric.Peer field a string. Additinoally, fixes calling the command without arguments and displaying the date in the right format. License: MIT Signed-off-by: Hector Sanjuan --- api/rest/client/methods.go | 6 +++- api/rest/client/request.go | 3 +- api/types.go | 53 ++++++++++++++++++++++++++++-- cluster.go | 5 ++- cmd/ipfs-cluster-ctl/formatters.go | 5 ++- cmd/ipfs-cluster-ctl/main.go | 15 +++++++-- monitor/metrics/store.go | 4 +-- monitor/metrics/util.go | 4 +-- rpc_api.go | 6 ++-- 9 files changed, 85 insertions(+), 16 deletions(-) diff --git a/api/rest/client/methods.go b/api/rest/client/methods.go index 2fcba6dc..2ddf36b5 100644 --- a/api/rest/client/methods.go +++ b/api/rest/client/methods.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/url" "os" @@ -205,9 +206,12 @@ func (c *defaultClient) GetConnectGraph() (api.ConnectGraphSerial, error) { return graphS, err } -// Metrics returns a map with the latest metrics of matching name +// Metrics returns a map with the latest valid metrics of the given name // for the current cluster peers. func (c *defaultClient) Metrics(name string) ([]api.Metric, error) { + if name == "" { + return nil, errors.New("bad metric name") + } var metrics []api.Metric err := c.do("GET", fmt.Sprintf("/monitor/metrics/%s", name), nil, nil, &metrics) return metrics, err diff --git a/api/rest/client/request.go b/api/rest/client/request.go index 5714ba4e..13a978ca 100644 --- a/api/rest/client/request.go +++ b/api/rest/client/request.go @@ -92,9 +92,10 @@ func (c *defaultClient) handleResponse(resp *http.Response, obj interface{}) err var apiErr api.Error err = json.Unmarshal(body, &apiErr) if err != nil { + // not json. 404s etc. return &api.Error{ Code: resp.StatusCode, - Message: err.Error(), + Message: string(body), } } return &apiErr diff --git a/api/types.go b/api/types.go index 8ee54be0..c4d0bb2d 100644 --- a/api/types.go +++ b/api/types.go @@ -9,6 +9,8 @@ package api import ( + "bytes" + "encoding/json" "fmt" "regexp" "sort" @@ -847,10 +849,10 @@ func (n *NodeWithMeta) Size() uint64 { // the Value, which should be interpreted by the PinAllocator. type Metric struct { Name string - Peer peer.ID // filled-in by Cluster. + Peer peer.ID Value string - Expire int64 // UnixNano - Valid bool // if the metric is not valid it will be discarded + Expire int64 + Valid bool } // SetTTL sets Metric to expire after the given time.Duration @@ -876,6 +878,51 @@ func (m *Metric) Discard() bool { return !m.Valid || m.Expired() } +// helper for JSON marshaling. The Metric type is already +// serializable, but not pretty to humans (API). +type metricSerial struct { + Name string `json:"name"` + Peer string `json:"peer"` + Value string `json:"value"` + Expire int64 `json:"expire"` + Valid bool `json:"valid"` +} + +// MarshalJSON allows a Metric to produce a JSON representation +// of itself. +func (m *Metric) MarshalJSON() ([]byte, error) { + return json.Marshal(&metricSerial{ + Name: m.Name, + Peer: peer.IDB58Encode(m.Peer), + Value: m.Value, + Expire: m.Expire, + }) +} + +// UnmarshalJSON decodes JSON on top of the Metric. +func (m *Metric) UnmarshalJSON(j []byte) error { + if bytes.Equal(j, []byte("null")) { + return nil + } + + ms := &metricSerial{} + err := json.Unmarshal(j, ms) + if err != nil { + return err + } + + p, err := peer.IDB58Decode(ms.Peer) + if err != nil { + return err + } + + m.Name = ms.Name + m.Peer = p + m.Value = ms.Value + m.Expire = ms.Expire + return nil +} + // Alert carries alerting information about a peer. WIP. type Alert struct { Peer peer.ID diff --git a/cluster.go b/cluster.go index 4982a3a3..0efa21a8 100644 --- a/cluster.go +++ b/cluster.go @@ -275,7 +275,10 @@ func (c *Cluster) alertsHandler() { // only the leader handles alerts leader, err := c.consensus.Leader() if err == nil && leader == c.id { - logger.Warningf("Peer %s received alert for %s in %s", c.id, alrt.MetricName, alrt.Peer.Pretty()) + logger.Warningf( + "Peer %s received alert for %s in %s", + c.id, alrt.MetricName, alrt.Peer, + ) switch alrt.MetricName { case pingMetricName: c.repinFromPeer(alrt.Peer) diff --git a/cmd/ipfs-cluster-ctl/formatters.go b/cmd/ipfs-cluster-ctl/formatters.go index 3d149784..3cd08168 100644 --- a/cmd/ipfs-cluster-ctl/formatters.go +++ b/cmd/ipfs-cluster-ctl/formatters.go @@ -6,8 +6,10 @@ import ( "fmt" "sort" "strings" + "time" "github.com/ipfs/ipfs-cluster/api" + peer "github.com/libp2p/go-libp2p-peer" ) func jsonFormatObject(resp interface{}) { @@ -216,7 +218,8 @@ func textFormatPrintAddedOutput(obj *api.AddedOutput) { } func textFormatPrintMetric(obj *api.Metric) { - fmt.Printf("%s: %s | Expire : %d\n", obj.Peer.Pretty(), obj.Value, obj.Expire) + date := time.Unix(0, obj.Expire).UTC().Format(time.RFC3339) + fmt.Printf("%s: %s | Expire: %s\n", peer.IDB58Encode(obj.Peer), obj.Value, date) } func textFormatPrintError(obj *api.Error) { diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index 02ed42c3..b69ca6a3 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -768,10 +768,21 @@ graph of the connections. Output is a dot file encoding the cluster's connectio Description: ` This commands displays the latest valid metrics of the given type logged by this peer for all current cluster peers. + +Currently supported metrics depend on the informer component used, +but usually are: + +- freespace +- ping `, - ArgsUsage: "Metric name", + ArgsUsage: "", Action: func(c *cli.Context) error { - resp, cerr := globalClient.Metrics(c.Args().First()) + metric := c.Args().First() + if metric == "" { + checkErr("", errors.New("provide a metric name")) + } + + resp, cerr := globalClient.Metrics(metric) formatResponse(c, resp, cerr) return nil }, diff --git a/monitor/metrics/store.go b/monitor/metrics/store.go index 34a73694..1bc1683c 100644 --- a/monitor/metrics/store.go +++ b/monitor/metrics/store.go @@ -71,14 +71,14 @@ func (mtrs *Store) Latest(name string) []api.Metric { // PeerMetrics returns the latest metrics for a given peer ID for // all known metrics types. It may return expired metrics. -func (mtrs *Store) PeerMetrics(peer peer.ID) []api.Metric { +func (mtrs *Store) PeerMetrics(pid peer.ID) []api.Metric { mtrs.mux.RLock() defer mtrs.mux.RUnlock() result := make([]api.Metric, 0) for _, byPeer := range mtrs.byName { - window, ok := byPeer[peer] + window, ok := byPeer[pid] if !ok { continue } diff --git a/monitor/metrics/util.go b/monitor/metrics/util.go index 17527027..79c77931 100644 --- a/monitor/metrics/util.go +++ b/monitor/metrics/util.go @@ -10,8 +10,8 @@ import ( // peerset func PeersetFilter(metrics []api.Metric, peerset []peer.ID) []api.Metric { peerMap := make(map[peer.ID]struct{}) - for _, peer := range peerset { - peerMap[peer] = struct{}{} + for _, pid := range peerset { + peerMap[pid] = struct{}{} } filtered := make([]api.Metric, 0, len(metrics)) diff --git a/rpc_api.go b/rpc_api.go index f6068ec3..5c553109 100644 --- a/rpc_api.go +++ b/rpc_api.go @@ -205,12 +205,12 @@ func (rpcapi *RPCAPI) BlockAllocate(ctx context.Context, in api.PinSerial, out * // Returned metrics are Valid and belong to current // Cluster peers. metrics := rpcapi.c.monitor.LatestMetrics(pingMetricName) - peers := make([]peer.ID, len(metrics), len(metrics)) + peers := make([]string, len(metrics), len(metrics)) for i, m := range metrics { - peers[i] = m.Peer + peers[i] = peer.IDB58Encode(m.Peer) } - *out = api.PeersToStrings(peers) + *out = peers return nil } From 85a7dc51145bbf6a261dae04b90a7c0e322674a1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sat, 27 Oct 2018 13:49:55 +0200 Subject: [PATCH 37/61] Allocate memory for response rpc types Might trigger a very weird race when pointing to nil License: MIT Signed-off-by: Hector Sanjuan --- cluster.go | 2 +- ipfsconn/ipfshttp/ipfshttp.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster.go b/cluster.go index 0efa21a8..1ac1b33f 100644 --- a/cluster.go +++ b/cluster.go @@ -1190,7 +1190,7 @@ func (c *Cluster) globalPinInfoCid(method string, h cid.Cid) (api.GlobalPinInfo, } func (c *Cluster) globalPinInfoSlice(method string) ([]api.GlobalPinInfo, error) { - var infos []api.GlobalPinInfo + infos := make([]api.GlobalPinInfo, 0) fullMap := make(map[string]api.GlobalPinInfo) members, err := c.consensus.Peers() diff --git a/ipfsconn/ipfshttp/ipfshttp.go b/ipfsconn/ipfshttp/ipfshttp.go index 288a797f..b6368901 100644 --- a/ipfsconn/ipfshttp/ipfshttp.go +++ b/ipfsconn/ipfshttp/ipfshttp.go @@ -328,7 +328,7 @@ func (ipfs *Connector) pinLsHandler(w http.ResponseWriter, r *http.Request) { Type: "recursive", } } else { - var pins []api.PinSerial + pins := make([]api.PinSerial, 0) err := ipfs.rpcClient.Call( "", "Cluster", @@ -429,7 +429,7 @@ func (ipfs *Connector) addHandler(w http.ResponseWriter, r *http.Request) { } func (ipfs *Connector) repoStatHandler(w http.ResponseWriter, r *http.Request) { - var peers []peer.ID + peers := make([]peer.ID, 0) err := ipfs.rpcClient.Call( "", "Cluster", @@ -770,7 +770,7 @@ func (ipfs *Connector) apiURL() string { func (ipfs *Connector) ConnectSwarms() error { ctx, cancel := context.WithTimeout(ipfs.ctx, ipfs.config.IPFSRequestTimeout) defer cancel() - var idsSerial []api.IDSerial + idsSerial := make([]api.IDSerial, 0) err := ipfs.rpcClient.Call( "", "Cluster", From d5fa2cf9f26774a462fe2d35fc838b58383ff156 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sat, 27 Oct 2018 02:38:28 +0200 Subject: [PATCH 38/61] fix race with go-libp2p-gorpc License: MIT Signed-off-by: Hector Sanjuan --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 366d7a86..564ea195 100644 --- a/package.json +++ b/package.json @@ -45,9 +45,9 @@ }, { "author": "hsanjuan", - "hash": "Qmas5ivzqmatd1tJWppjcupYY8YS2DJHJZxSMsMoLbCQ2R", + "hash": "QmPYiV9nwnXPxdn9zDgY4d9yaHwTS414sUb1K6nvQVHqqo", "name": "go-libp2p-gorpc", - "version": "1.0.23" + "version": "1.0.24" }, { "author": "libp2p", From 765987ef2ce8e584e864d37f8fb28e20e40be0c5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sat, 27 Oct 2018 01:54:07 +0200 Subject: [PATCH 39/61] Improvements License: MIT Signed-off-by: Hector Sanjuan --- cluster.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cluster.go b/cluster.go index 1ac1b33f..834344da 100644 --- a/cluster.go +++ b/cluster.go @@ -1093,11 +1093,12 @@ func (c *Cluster) Peers() []api.ID { logger.Error("an empty list of peers will be returned") return []api.ID{} } + lenMembers := len(members) - peersSerial := make([]api.IDSerial, len(members), len(members)) - peers := make([]api.ID, len(members), len(members)) + peersSerial := make([]api.IDSerial, lenMembers, lenMembers) + peers := make([]api.ID, lenMembers, lenMembers) - ctxs, cancels := rpcutil.CtxsWithCancel(c.ctx, len(members)) + ctxs, cancels := rpcutil.CtxsWithCancel(c.ctx, lenMembers) defer rpcutil.MultiCancel(cancels) errs := c.rpcClient.MultiCall( @@ -1133,13 +1134,14 @@ func (c *Cluster) globalPinInfoCid(method string, h cid.Cid) (api.GlobalPinInfo, logger.Error(err) return api.GlobalPinInfo{}, err } + lenMembers := len(members) - replies := make([]api.PinInfoSerial, len(members), len(members)) + replies := make([]api.PinInfoSerial, lenMembers, lenMembers) arg := api.Pin{ Cid: h, } - ctxs, cancels := rpcutil.CtxsWithCancel(c.ctx, len(members)) + ctxs, cancels := rpcutil.CtxsWithCancel(c.ctx, lenMembers) defer rpcutil.MultiCancel(cancels) errs := c.rpcClient.MultiCall( @@ -1198,10 +1200,11 @@ func (c *Cluster) globalPinInfoSlice(method string) ([]api.GlobalPinInfo, error) logger.Error(err) return []api.GlobalPinInfo{}, err } + lenMembers := len(members) - replies := make([][]api.PinInfoSerial, len(members), len(members)) + replies := make([][]api.PinInfoSerial, lenMembers, lenMembers) - ctxs, cancels := rpcutil.CtxsWithCancel(c.ctx, len(members)) + ctxs, cancels := rpcutil.CtxsWithCancel(c.ctx, lenMembers) defer rpcutil.MultiCancel(cancels) errs := c.rpcClient.MultiCall( From d63a5e26678ff450a22296ac5736b0cbdcdc05c2 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sat, 27 Oct 2018 00:40:06 +0200 Subject: [PATCH 40/61] Fix race on ApplyTo The FSM tries to decode an operation on top of the *LogOp. We might still be using the *LogOp.Cid.Allocations slice. We need to make a deep of *LogOp.Cid before returning from ApplyTo. This one was tricky... License: MIT Signed-off-by: Hector Sanjuan --- api/types.go | 9 +++++++++ consensus/raft/log_op.go | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/api/types.go b/api/types.go index c4d0bb2d..c5ba10cb 100644 --- a/api/types.go +++ b/api/types.go @@ -829,6 +829,15 @@ func (pins PinSerial) ToPin() Pin { } } +// Clone returns a deep copy of the PinSerial. +func (pins PinSerial) Clone() PinSerial { + new := pins // this copy all the simple fields. + // slices are pointers. We need to explicitally copy them. + new.Allocations = make([]string, len(pins.Allocations)) + copy(new.Allocations, pins.Allocations) + return new +} + // NodeWithMeta specifies a block of data and a set of optional metadata fields // carrying information about the encoded ipld node type NodeWithMeta struct { diff --git a/consensus/raft/log_op.go b/consensus/raft/log_op.go index f4f26af3..b0095db3 100644 --- a/consensus/raft/log_op.go +++ b/consensus/raft/log_op.go @@ -36,9 +36,18 @@ func (op *LogOp) ApplyTo(cstate consensus.State) (consensus.State, error) { panic("received unexpected state type") } + // Copy the Cid. We are about to pass it to go-routines + // that will make things with it (read its fields). However, + // as soon as ApplyTo is done, the next operation will be deserealized + // on top of "op". This can cause data races with the slices in + // api.PinSerial, which don't get copied when passed. + pinS := op.Cid.Clone() + + pin := pinS.ToPin() + switch op.Type { case LogOpPin: - err = state.Add(op.Cid.ToPin()) + err = state.Add(pin) if err != nil { goto ROLLBACK } @@ -46,11 +55,11 @@ func (op *LogOp) ApplyTo(cstate consensus.State) (consensus.State, error) { op.consensus.rpcClient.Go("", "Cluster", "Track", - op.Cid, + pinS, &struct{}{}, nil) case LogOpUnpin: - err = state.Rm(op.Cid.ToPin().Cid) + err = state.Rm(pin.Cid) if err != nil { goto ROLLBACK } @@ -58,7 +67,7 @@ func (op *LogOp) ApplyTo(cstate consensus.State) (consensus.State, error) { op.consensus.rpcClient.Go("", "Cluster", "Untrack", - op.Cid, + pinS, &struct{}{}, nil) default: From ca3fe646b11b21acdf114df78dc39a1c5a784971 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 26 Oct 2018 14:45:16 +0200 Subject: [PATCH 41/61] Fix race condition when shutting down and watchPeers() run License: MIT Signed-off-by: Hector Sanjuan --- cluster.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster.go b/cluster.go index 834344da..7ca3bffa 100644 --- a/cluster.go +++ b/cluster.go @@ -313,6 +313,8 @@ func (c *Cluster) watchPeers() { } if !hasMe { + c.shutdownLock.Lock() + defer c.shutdownLock.Unlock() logger.Infof("%s: removed from raft. Initiating shutdown", c.id.Pretty()) c.removed = true go c.Shutdown() From de2848fb2369ac3cca2226a62eb49b72c15ffc02 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 29 Oct 2018 12:50:36 +0100 Subject: [PATCH 42/61] Upgrade ipld-related dependencies. Remove impoters DAG forced-Batching. License: MIT Signed-off-by: Hector Sanjuan --- adder/ipfsadd/add.go | 2 ++ package.json | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/adder/ipfsadd/add.go b/adder/ipfsadd/add.go index 9c47c403..16b46055 100644 --- a/adder/ipfsadd/add.go +++ b/adder/ipfsadd/add.go @@ -91,6 +91,8 @@ func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { return nil, err } + // Cluster: we don't do batching. + params := ihelper.DagBuilderParams{ Dagserv: adder.dagService, RawLeaves: adder.RawLeaves, diff --git a/package.json b/package.json index 366d7a86..dd5b59d9 100644 --- a/package.json +++ b/package.json @@ -105,9 +105,9 @@ }, { "author": "hector", - "hash": "QmPG32VXR5jmpo9q8R9FNdR4Ae97Ky9CiZE6SctJLUB79H", + "hash": "QmQyUyYcpKG1u53V7N25qRTGw5XwaAxTMKXbduqHotQztg", "name": "go-ipfs-posinfo", - "version": "0.1.0" + "version": "0.1.2" }, { "author": "dustin", @@ -117,15 +117,15 @@ }, { "author": "why", - "hash": "QmTJUySFxXjh54zEoFbzQEmGD3yj89XKS3A28y7Nqsn1TC", + "hash": "QmfB3oNXGGq9S4B2a9YeCajoATms3Zw2VvDm8fK7VeLSV8", "name": "go-unixfs", - "version": "1.1.15" + "version": "1.1.16" }, { "author": "why", - "hash": "QmY8BMUSpCwNiTmFhACmC9Bt1qT63cHP35AoQAus4x14qH", + "hash": "QmSei8kFMfqdJq7Q68d2LMnHbTWKKg2daA29ezUYFAUNgc", "name": "go-merkledag", - "version": "1.1.13" + "version": "1.1.15" }, { "hash": "QmQHnqaNULV8WeUGgh97o9K3KAW6kWQmDyNf9UuikgnPTe", @@ -134,9 +134,9 @@ }, { "author": "hsanjuan", - "hash": "QmcctvPwYdaJvM4VefQnKAfU2ppSVfjpECvBVVEm18qk68", + "hash": "QmTFCS9QwPQEihtjqi6zTX5n21J5y1yQwc3hnk7t3hZDLN", "name": "go-mfs", - "version": "0.1.15" + "version": "0.1.17" }, { "author": "blang", From 0b923f8d49f0311b24ac6d9839d82b6cc11abc93 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 29 Oct 2018 14:53:59 +0100 Subject: [PATCH 43/61] Set envconfig version (more or less), and move setDefaults down a bit. License: MIT Signed-off-by: Hector Sanjuan --- cluster_config.go | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster_config.go b/cluster_config.go index c9268771..6d81c6aa 100644 --- a/cluster_config.go +++ b/cluster_config.go @@ -273,8 +273,6 @@ func (cfg *Config) setDefaults() { // sets the Config fields from it. Note that it should be JSON // as generated by ToJSON(). func (cfg *Config) LoadJSON(raw []byte) error { - cfg.setDefaults() - jcfg := &configJSON{} err := json.Unmarshal(raw, jcfg) if err != nil { @@ -282,6 +280,8 @@ func (cfg *Config) LoadJSON(raw []byte) error { return err } + cfg.setDefaults() + if jcfg.Peers != nil || jcfg.Bootstrap != nil { logger.Error(` Your configuration is using cluster.Peers and/or cluster.Bootstrap diff --git a/package.json b/package.json index 3522d994..94af4603 100644 --- a/package.json +++ b/package.json @@ -152,9 +152,9 @@ }, { "author": "lanzafame", - "hash": "QmVPrFJ6dYJyt7uULgE7wouhvXDzYBZVjYvjMCE5Uqxgfy", + "hash": "QmYgGtLm9WJRgh6iuaZap8qVC1gqixFbZCNfhjLNBhWMCm", "name": "envconfig", - "version": "0.0.0" + "version": "1.3.1" } ], "gxVersion": "0.11.0", From 8a045ae2d2fa25df8c540f6335a81989947df295 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 29 Oct 2018 17:25:52 +0100 Subject: [PATCH 44/61] Update to latest go-libp2p-pubsub (with renames) License: MIT Signed-off-by: Hector Sanjuan --- monitor/pubsubmon/pubsubmon.go | 9 +++++---- monitor/pubsubmon/pubsubmon_test.go | 4 ++-- package.json | 12 ++++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/monitor/pubsubmon/pubsubmon.go b/monitor/pubsubmon/pubsubmon.go index 439f2990..617d948b 100644 --- a/monitor/pubsubmon/pubsubmon.go +++ b/monitor/pubsubmon/pubsubmon.go @@ -5,16 +5,17 @@ package pubsubmon import ( "bytes" "context" + "sync" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/monitor/metrics" logging "github.com/ipfs/go-log" - floodsub "github.com/libp2p/go-floodsub" rpc "github.com/libp2p/go-libp2p-gorpc" host "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" + pubsub "github.com/libp2p/go-libp2p-pubsub" msgpack "github.com/multiformats/go-multicodec/msgpack" ) @@ -34,8 +35,8 @@ type Monitor struct { rpcReady chan struct{} host host.Host - pubsub *floodsub.PubSub - subscription *floodsub.Subscription + pubsub *pubsub.PubSub + subscription *pubsub.Subscription metrics *metrics.Store checker *metrics.Checker @@ -59,7 +60,7 @@ func New(h host.Host, cfg *Config) (*Monitor, error) { mtrs := metrics.NewStore() checker := metrics.NewChecker(mtrs) - pubsub, err := floodsub.NewGossipSub(ctx, h) + pubsub, err := pubsub.NewGossipSub(ctx, h) if err != nil { cancel() return nil, err diff --git a/monitor/pubsubmon/pubsubmon_test.go b/monitor/pubsubmon/pubsubmon_test.go index 4190c43e..eb12b290 100644 --- a/monitor/pubsubmon/pubsubmon_test.go +++ b/monitor/pubsubmon/pubsubmon_test.go @@ -9,10 +9,10 @@ import ( "testing" "time" - floodsub "github.com/libp2p/go-floodsub" libp2p "github.com/libp2p/go-libp2p" peer "github.com/libp2p/go-libp2p-peer" peerstore "github.com/libp2p/go-libp2p-peerstore" + pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/test" @@ -21,7 +21,7 @@ import ( func init() { // GossipSub needs to heartbeat to discover newly connected hosts // This speeds things up a little. - floodsub.GossipSubHeartbeatInterval = 50 * time.Millisecond + pubsub.GossipSubHeartbeatInterval = 50 * time.Millisecond } type metricFactory struct { diff --git a/package.json b/package.json index ce5f11b4..b9383738 100644 --- a/package.json +++ b/package.json @@ -79,12 +79,6 @@ "name": "go-ipfs-api", "version": "1.4.2" }, - { - "author": "whyrusleeping", - "hash": "Qmc71UCoZWor2ADQAWjNnrSVgEtPpEsQbCXf9Gs7WWz7MV", - "name": "go-libp2p-floodsub", - "version": "100.10.2" - }, { "author": "whyrusleeping", "hash": "QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7", @@ -155,6 +149,12 @@ "hash": "QmYgGtLm9WJRgh6iuaZap8qVC1gqixFbZCNfhjLNBhWMCm", "name": "envconfig", "version": "1.3.1" + }, + { + "author": "whyrusleeping", + "hash": "QmYmrxfax5xGfLF6SL2Bq7SDEzFZFyNcLvGi8ExdC5iiko", + "name": "go-libp2p-pubsub", + "version": "100.11.3" } ], "gxVersion": "0.11.0", From 7c90e550dce0ca3811da83e6643bc4c84fcd3074 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 29 Oct 2018 18:03:04 +0100 Subject: [PATCH 45/61] Release 0.7.0-rc1 License: MIT Signed-off-by: Hector Sanjuan --- cmd/ipfs-cluster-ctl/main.go | 2 +- version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index b69ca6a3..a8ab6805 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -27,7 +27,7 @@ const programName = `ipfs-cluster-ctl` // Version is the cluster-ctl tool version. It should match // the IPFS cluster's version -const Version = "0.6.0" +const Version = "0.7.0-rc1" var ( defaultHost = "/ip4/127.0.0.1/tcp/9094" diff --git a/version.go b/version.go index b5c3e710..f2348694 100644 --- a/version.go +++ b/version.go @@ -9,7 +9,7 @@ import ( // Version is the current cluster version. Version alignment between // components, apis and tools ensures compatibility among them. -var Version = semver.MustParse("0.6.0") +var Version = semver.MustParse("0.7.0-rc1") // RPCProtocol is used to send libp2p messages between cluster peers var RPCProtocol = protocol.ID( From e787773ed4228cbfe054dafbbb5402eb76cb6cc6 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 29 Oct 2018 18:03:58 +0100 Subject: [PATCH 46/61] gx publish 0.7.0-rc1 License: MIT Signed-off-by: Hector Sanjuan --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index 314ad027..3fb5af15 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -0.6.0: QmUY1LMFsjmVn1prwerDFJyGpbatfbABj7xGmSCQTL56bM +0.7.0-rc1: QmUN1SnZ8w8E2AUJzK1VM9ASFdxgHsXEJJXvt7fRcxufnz diff --git a/package.json b/package.json index b9383738..93b8652e 100644 --- a/package.json +++ b/package.json @@ -162,6 +162,6 @@ "license": "MIT", "name": "ipfs-cluster", "releaseCmd": "git commit -S -a -m \"gx publish $VERSION\"", - "version": "0.6.0" + "version": "0.7.0-rc1" } From 1bc7f5a6437502cd936c8e66781ad48b02edaef2 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 30 Oct 2018 11:54:32 +0100 Subject: [PATCH 47/61] Re-order shutdownLock fields in cluster declaration License: MIT Signed-off-by: Hector Sanjuan --- cluster.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cluster.go b/cluster.go index 7ca3bffa..03f8dbbd 100644 --- a/cluster.go +++ b/cluster.go @@ -56,15 +56,18 @@ type Cluster struct { allocator PinAllocator informer Informer + doneCh chan struct{} + readyCh chan struct{} + readyB bool + wg sync.WaitGroup + + // peerAdd + paMux sync.Mutex + + // shutdown function and related variables shutdownLock sync.Mutex shutdownB bool removed bool - doneCh chan struct{} - readyCh chan struct{} - readyB bool - wg sync.WaitGroup - - paMux sync.Mutex } // NewCluster builds a new IPFS Cluster peer. It initializes a LibP2P host, From 820df17359c506d3c0062e49adbf01ce075ac3b7 Mon Sep 17 00:00:00 2001 From: Adrian Lanzafame Date: Tue, 30 Oct 2018 12:00:38 +1000 Subject: [PATCH 48/61] add pinSerial.DecodeCid and benchs License: MIT Signed-off-by: Adrian Lanzafame --- api/types.go | 10 ++++++++++ api/types_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/api/types.go b/api/types.go index c5ba10cb..23ce1593 100644 --- a/api/types.go +++ b/api/types.go @@ -838,6 +838,16 @@ func (pins PinSerial) Clone() PinSerial { return new } +// DecodeCid retrieves just the cid from a PinSerial without +// allocating a Pin. +func (pins PinSerial) DecodeCid() cid.Cid { + c, err := cid.Decode(pins.Cid) + if err != nil { + logger.Debug(pins.Cid, err) + } + return c +} + // NodeWithMeta specifies a block of data and a set of optional metadata fields // carrying information about the encoded ipld node type NodeWithMeta struct { diff --git a/api/types_test.go b/api/types_test.go index 463ab670..dc56d091 100644 --- a/api/types_test.go +++ b/api/types_test.go @@ -257,3 +257,45 @@ func TestMetric(t *testing.T) { t.Error("looks like a bad ttl") } } + +func BenchmarkPinSerial_ToPin(b *testing.B) { + pin := Pin{ + Cid: testCid1, + Type: ClusterDAGType, + Allocations: []peer.ID{testPeerID1}, + Reference: testCid2, + MaxDepth: -1, + PinOptions: PinOptions{ + ReplicationFactorMax: -1, + ReplicationFactorMin: -1, + Name: "A test pin", + }, + } + pinS := pin.ToSerial() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + pinS.ToPin() + } +} + +func BenchmarkPinSerial_DecodeCid(b *testing.B) { + pin := Pin{ + Cid: testCid1, + Type: ClusterDAGType, + Allocations: []peer.ID{testPeerID1}, + Reference: testCid2, + MaxDepth: -1, + PinOptions: PinOptions{ + ReplicationFactorMax: -1, + ReplicationFactorMin: -1, + Name: "A test pin", + }, + } + pinS := pin.ToSerial() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + pinS.DecodeCid() + } +} From a89a6da45b61109d32edd718eac2cce8244a1c2c Mon Sep 17 00:00:00 2001 From: Adrian Lanzafame Date: Tue, 30 Oct 2018 12:01:27 +1000 Subject: [PATCH 49/61] use DecodeCid in rpc_api License: MIT Signed-off-by: Adrian Lanzafame --- rpc_api.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/rpc_api.go b/rpc_api.go index 5c553109..c5a2c703 100644 --- a/rpc_api.go +++ b/rpc_api.go @@ -37,7 +37,7 @@ func (rpcapi *RPCAPI) Pin(ctx context.Context, in api.PinSerial, out *struct{}) // Unpin runs Cluster.Unpin(). func (rpcapi *RPCAPI) Unpin(ctx context.Context, in api.PinSerial, out *struct{}) error { - c := in.ToPin().Cid + c := in.DecodeCid() return rpcapi.c.Unpin(c) } @@ -124,7 +124,7 @@ func (rpcapi *RPCAPI) StatusAllLocal(ctx context.Context, in struct{}, out *[]ap // Status runs Cluster.Status(). func (rpcapi *RPCAPI) Status(ctx context.Context, in api.PinSerial, out *api.GlobalPinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo, err := rpcapi.c.Status(c) *out = pinfo.ToSerial() return err @@ -132,7 +132,7 @@ func (rpcapi *RPCAPI) Status(ctx context.Context, in api.PinSerial, out *api.Glo // StatusLocal runs Cluster.StatusLocal(). func (rpcapi *RPCAPI) StatusLocal(ctx context.Context, in api.PinSerial, out *api.PinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo := rpcapi.c.StatusLocal(c) *out = pinfo.ToSerial() return nil @@ -154,7 +154,7 @@ func (rpcapi *RPCAPI) SyncAllLocal(ctx context.Context, in struct{}, out *[]api. // Sync runs Cluster.Sync(). func (rpcapi *RPCAPI) Sync(ctx context.Context, in api.PinSerial, out *api.GlobalPinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo, err := rpcapi.c.Sync(c) *out = pinfo.ToSerial() return err @@ -162,7 +162,7 @@ func (rpcapi *RPCAPI) Sync(ctx context.Context, in api.PinSerial, out *api.Globa // SyncLocal runs Cluster.SyncLocal(). func (rpcapi *RPCAPI) SyncLocal(ctx context.Context, in api.PinSerial, out *api.PinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo, err := rpcapi.c.SyncLocal(c) *out = pinfo.ToSerial() return err @@ -177,7 +177,7 @@ func (rpcapi *RPCAPI) RecoverAllLocal(ctx context.Context, in struct{}, out *[]a // Recover runs Cluster.Recover(). func (rpcapi *RPCAPI) Recover(ctx context.Context, in api.PinSerial, out *api.GlobalPinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo, err := rpcapi.c.Recover(c) *out = pinfo.ToSerial() return err @@ -185,7 +185,7 @@ func (rpcapi *RPCAPI) Recover(ctx context.Context, in api.PinSerial, out *api.Gl // RecoverLocal runs Cluster.RecoverLocal(). func (rpcapi *RPCAPI) RecoverLocal(ctx context.Context, in api.PinSerial, out *api.PinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo, err := rpcapi.c.RecoverLocal(c) *out = pinfo.ToSerial() return err @@ -248,7 +248,7 @@ func (rpcapi *RPCAPI) Track(ctx context.Context, in api.PinSerial, out *struct{} // Untrack runs PinTracker.Untrack(). func (rpcapi *RPCAPI) Untrack(ctx context.Context, in api.PinSerial, out *struct{}) error { - c := in.ToPin().Cid + c := in.DecodeCid() return rpcapi.c.tracker.Untrack(c) } @@ -260,7 +260,7 @@ func (rpcapi *RPCAPI) TrackerStatusAll(ctx context.Context, in struct{}, out *[] // TrackerStatus runs PinTracker.Status(). func (rpcapi *RPCAPI) TrackerStatus(ctx context.Context, in api.PinSerial, out *api.PinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo := rpcapi.c.tracker.Status(c) *out = pinfo.ToSerial() return nil @@ -275,7 +275,7 @@ func (rpcapi *RPCAPI) TrackerRecoverAll(ctx context.Context, in struct{}, out *[ // TrackerRecover runs PinTracker.Recover(). func (rpcapi *RPCAPI) TrackerRecover(ctx context.Context, in api.PinSerial, out *api.PinInfoSerial) error { - c := in.ToPin().Cid + c := in.DecodeCid() pinfo, err := rpcapi.c.tracker.Recover(c) *out = pinfo.ToSerial() return err @@ -287,20 +287,20 @@ func (rpcapi *RPCAPI) TrackerRecover(ctx context.Context, in api.PinSerial, out // IPFSPin runs IPFSConnector.Pin(). func (rpcapi *RPCAPI) IPFSPin(ctx context.Context, in api.PinSerial, out *struct{}) error { - c := in.ToPin().Cid + c := in.DecodeCid() depth := in.ToPin().MaxDepth return rpcapi.c.ipfs.Pin(ctx, c, depth) } // IPFSUnpin runs IPFSConnector.Unpin(). func (rpcapi *RPCAPI) IPFSUnpin(ctx context.Context, in api.PinSerial, out *struct{}) error { - c := in.ToPin().Cid + c := in.DecodeCid() return rpcapi.c.ipfs.Unpin(ctx, c) } // IPFSPinLsCid runs IPFSConnector.PinLsCid(). func (rpcapi *RPCAPI) IPFSPinLsCid(ctx context.Context, in api.PinSerial, out *api.IPFSPinStatus) error { - c := in.ToPin().Cid + c := in.DecodeCid() b, err := rpcapi.c.ipfs.PinLsCid(ctx, c) *out = b return err @@ -347,7 +347,7 @@ func (rpcapi *RPCAPI) IPFSBlockPut(ctx context.Context, in api.NodeWithMeta, out // IPFSBlockGet runs IPFSConnector.BlockGet(). func (rpcapi *RPCAPI) IPFSBlockGet(ctx context.Context, in api.PinSerial, out *[]byte) error { - c := in.ToPin().Cid + c := in.DecodeCid() res, err := rpcapi.c.ipfs.BlockGet(c) *out = res return err From 91358e1ed139b495d127c29613726ff3d47e1282 Mon Sep 17 00:00:00 2001 From: Adrian Lanzafame Date: Tue, 30 Oct 2018 12:02:15 +1000 Subject: [PATCH 50/61] only call ToPin when absolutely required License: MIT Signed-off-by: Adrian Lanzafame --- consensus/raft/log_op.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/consensus/raft/log_op.go b/consensus/raft/log_op.go index b0095db3..e5f86c20 100644 --- a/consensus/raft/log_op.go +++ b/consensus/raft/log_op.go @@ -43,11 +43,9 @@ func (op *LogOp) ApplyTo(cstate consensus.State) (consensus.State, error) { // api.PinSerial, which don't get copied when passed. pinS := op.Cid.Clone() - pin := pinS.ToPin() - switch op.Type { case LogOpPin: - err = state.Add(pin) + err = state.Add(pinS.ToPin()) if err != nil { goto ROLLBACK } From 4f194f52d3bf3248d193e38240993e7e2dbba21f Mon Sep 17 00:00:00 2001 From: Adrian Lanzafame Date: Tue, 30 Oct 2018 12:02:58 +1000 Subject: [PATCH 51/61] use DecodeCid in log_op License: MIT Signed-off-by: Adrian Lanzafame --- consensus/raft/log_op.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/consensus/raft/log_op.go b/consensus/raft/log_op.go index e5f86c20..f9a10e51 100644 --- a/consensus/raft/log_op.go +++ b/consensus/raft/log_op.go @@ -50,24 +50,28 @@ func (op *LogOp) ApplyTo(cstate consensus.State) (consensus.State, error) { goto ROLLBACK } // Async, we let the PinTracker take care of any problems - op.consensus.rpcClient.Go("", + op.consensus.rpcClient.Go( + "", "Cluster", "Track", pinS, &struct{}{}, - nil) + nil, + ) case LogOpUnpin: - err = state.Rm(pin.Cid) + err = state.Rm(pinS.DecodeCid()) if err != nil { goto ROLLBACK } // Async, we let the PinTracker take care of any problems - op.consensus.rpcClient.Go("", + op.consensus.rpcClient.Go( + "", "Cluster", "Untrack", pinS, &struct{}{}, - nil) + nil, + ) default: logger.Error("unknown LogOp type. Ignoring") } From a7787029cb2fa50d6308ba8f9ef0bebf82a26587 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 30 Oct 2018 16:33:05 +0100 Subject: [PATCH 52/61] Fix #600: Remote pins should not error License: MIT Signed-off-by: Hector Sanjuan --- pintracker/optracker/operationtracker.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pintracker/optracker/operationtracker.go b/pintracker/optracker/operationtracker.go index 5962e067..e469753a 100644 --- a/pintracker/optracker/operationtracker.go +++ b/pintracker/optracker/operationtracker.go @@ -94,6 +94,7 @@ func (opt *OperationTracker) Status(c cid.Cid) (api.TrackerStatus, bool) { // SetError transitions an operation for a Cid into PhaseError if its Status // is PhaseDone. Any other phases are considered in-flight and not touched. // For things already in error, the error message is updated. +// Remote pins are ignored too. func (opt *OperationTracker) SetError(c cid.Cid, err error) { opt.mu.Lock() defer opt.mu.Unlock() @@ -102,7 +103,14 @@ func (opt *OperationTracker) SetError(c cid.Cid, err error) { return } - if ph := op.Phase(); ph == PhaseDone || ph == PhaseError { + ph := op.Phase() + ty := op.Type() + + if ty == OperationRemote { + return + } + + if ph == PhaseDone || ph == PhaseError { op.SetPhase(PhaseError) op.SetError(err) } From f9a3d46258a16cd75a420ef336f13d3cf3274842 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 31 Oct 2018 17:12:12 +0100 Subject: [PATCH 53/61] Update gx and gx-go to latest stable License: MIT Signed-off-by: Hector Sanjuan --- deptools/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deptools/Makefile b/deptools/Makefile index 589106b0..21aef7ae 100644 --- a/deptools/Makefile +++ b/deptools/Makefile @@ -1,5 +1,5 @@ -gx_version=v0.13.0 -gx-go_version=v1.8.0 +gx_version=v0.14.1 +gx-go_version=v1.9.0 gateway=https://ipfs.io local_gateway=http://127.0.0.1:8080 dist=dist.ipfs.io From 91ddce49632eb6f4c382b30cf202b6471ce7e386 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 10:36:14 +0100 Subject: [PATCH 54/61] Fix the paths handled by the proxy License: MIT Signed-off-by: Hector Sanjuan --- ipfsconn/ipfshttp/ipfshttp.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ipfsconn/ipfshttp/ipfshttp.go b/ipfsconn/ipfshttp/ipfshttp.go index b6368901..47f16b6f 100644 --- a/ipfsconn/ipfshttp/ipfshttp.go +++ b/ipfsconn/ipfshttp/ipfshttp.go @@ -191,14 +191,14 @@ func NewConnector(cfg *Config) (*Connector, error) { } smux.Handle("/", proxyHandler) - smux.HandleFunc("/api/v0/pin/add/", ipfs.pinHandler) - smux.HandleFunc("/api/v0/pin/rm/", ipfs.unpinHandler) - smux.HandleFunc("/api/v0/pin/ls", ipfs.pinLsHandler) // required to handle /pin/ls for all pins - smux.HandleFunc("/api/v0/pin/ls/", ipfs.pinLsHandler) + smux.HandleFunc("/api/v0/pin/add", ipfs.pinHandler) // add?arg=xxx + smux.HandleFunc("/api/v0/pin/add/", ipfs.pinHandler) // add/xxx + smux.HandleFunc("/api/v0/pin/rm", ipfs.unpinHandler) // rm?arg=xxx + smux.HandleFunc("/api/v0/pin/rm/", ipfs.unpinHandler) // rm/xxx + smux.HandleFunc("/api/v0/pin/ls", ipfs.pinLsHandler) // required to handle /pin/ls for all pins + smux.HandleFunc("/api/v0/pin/ls/", ipfs.pinLsHandler) // ls/xxx smux.HandleFunc("/api/v0/add", ipfs.addHandler) - smux.HandleFunc("/api/v0/add/", ipfs.addHandler) smux.HandleFunc("/api/v0/repo/stat", ipfs.repoStatHandler) - smux.HandleFunc("/api/v0/repo/stat/", ipfs.repoStatHandler) go ipfs.run() return ipfs, nil From 74311c5969beba2a5d9168a188d3fa08ecb6e86d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 11:12:38 +0100 Subject: [PATCH 55/61] Address review comments Remove unused code from tests License: MIT Signed-off-by: Hector Sanjuan --- pintracker/optracker/operationtracker.go | 7 +- pintracker/pintracker_test.go | 140 +++++++++-------------- 2 files changed, 53 insertions(+), 94 deletions(-) diff --git a/pintracker/optracker/operationtracker.go b/pintracker/optracker/operationtracker.go index e469753a..43902264 100644 --- a/pintracker/optracker/operationtracker.go +++ b/pintracker/optracker/operationtracker.go @@ -103,14 +103,11 @@ func (opt *OperationTracker) SetError(c cid.Cid, err error) { return } - ph := op.Phase() - ty := op.Type() - - if ty == OperationRemote { + if ty := op.Type(); ty == OperationRemote { return } - if ph == PhaseDone || ph == PhaseError { + if ph := op.Phase(); ph == PhaseDone || ph == PhaseError { op.SetPhase(PhaseError) op.SetError(err) } diff --git a/pintracker/pintracker_test.go b/pintracker/pintracker_test.go index 637c6a8a..11ffa997 100644 --- a/pintracker/pintracker_test.go +++ b/pintracker/pintracker_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + peer "github.com/libp2p/go-libp2p-peer" + ipfscluster "github.com/ipfs/ipfs-cluster" "github.com/ipfs/ipfs-cluster/api" "github.com/ipfs/ipfs-cluster/pintracker/maptracker" @@ -17,7 +19,6 @@ import ( cid "github.com/ipfs/go-cid" rpc "github.com/libp2p/go-libp2p-gorpc" - peer "github.com/libp2p/go-libp2p-peer" ) var ( @@ -60,6 +61,9 @@ func (mock *mockService) IPFSPinLsCid(ctx context.Context, in api.PinSerial, out switch in.Cid { case test.TestCid1, test.TestCid2: *out = api.IPFSPinStatusRecursive + case test.TestCid4: + *out = api.IPFSPinStatusError + return errors.New("an ipfs error") default: *out = api.IPFSPinStatusUnpinned } @@ -85,94 +89,6 @@ func (mock *mockService) IPFSPinLs(ctx context.Context, in string, out *map[stri return nil } -func (mock *mockService) Status(ctx context.Context, in api.PinSerial, out *api.GlobalPinInfoSerial) error { - switch in.Cid { - case test.ErrorCid: - return test.ErrBadCid - case test.TestCid1: - c1, _ := cid.Decode(test.TestCid1) - *out = api.GlobalPinInfo{ - Cid: c1, - PeerMap: map[peer.ID]api.PinInfo{ - test.TestPeerID1: { - Cid: c1, - Peer: test.TestPeerID1, - Status: api.TrackerStatusPinned, - TS: time.Now(), - }, - }, - }.ToSerial() - // case test.TestSlowCid1: - // sc1 := test.MustDecodeCid(test.TestSlowCid1) - // *out = api.GlobalPinInfo{ - // Cid: sc1, - // PeerMap: map[peer.ID]api.PinInfo{ - // test.TestPeerID1: { - // Cid: sc1, - // Peer: test.TestPeerID1, - // Status: api.TrackerStatusPinned, - // TS: time.Now(), - // }, - // }, - // }.ToSerial() - } - return nil -} - -func (mock *mockService) StatusAll(ctx context.Context, in struct{}, out *[]api.GlobalPinInfoSerial) error { - c1, _ := cid.Decode(test.TestCid1) - c2, _ := cid.Decode(test.TestCid2) - c3, _ := cid.Decode(test.TestCid3) - slowC1 := test.MustDecodeCid(test.TestSlowCid1) - *out = ipfscluster.GlobalPinInfoSliceToSerial([]api.GlobalPinInfo{ - { - Cid: c1, - PeerMap: map[peer.ID]api.PinInfo{ - test.TestPeerID1: { - Cid: c1, - Peer: test.TestPeerID1, - Status: api.TrackerStatusPinned, - TS: time.Now(), - }, - }, - }, - { - Cid: c2, - PeerMap: map[peer.ID]api.PinInfo{ - test.TestPeerID1: { - Cid: c2, - Peer: test.TestPeerID1, - Status: api.TrackerStatusPinning, - TS: time.Now(), - }, - }, - }, - { - Cid: c3, - PeerMap: map[peer.ID]api.PinInfo{ - test.TestPeerID1: { - Cid: c3, - Peer: test.TestPeerID1, - Status: api.TrackerStatusPinError, - TS: time.Now(), - }, - }, - }, - { - Cid: slowC1, - PeerMap: map[peer.ID]api.PinInfo{ - test.TestPeerID1: { - Cid: slowC1, - Peer: test.TestPeerID1, - Status: api.TrackerStatusPinning, - TS: time.Now(), - }, - }, - }, - }) - return nil -} - func (mock *mockService) Pins(ctx context.Context, in struct{}, out *[]api.PinSerial) error { *out = []api.PinSerial{ api.PinWithOpts(test.MustDecodeCid(test.TestCid1), pinOpts).ToSerial(), @@ -1068,3 +984,49 @@ func TestTrackUntrackWithCancel(t *testing.T) { }) } } + +func TestPinTracker_RemoteIgnoresError(t *testing.T) { + testF := func(t *testing.T, pt ipfscluster.PinTracker) { + remoteCid := test.MustDecodeCid(test.TestCid4) + + remote := api.PinWithOpts(remoteCid, pinOpts) + remote.Allocations = []peer.ID{test.TestPeerID2} + remote.ReplicationFactorMin = 1 + remote.ReplicationFactorMax = 1 + + err := pt.Track(remote) + if err != nil { + t.Fatal(err) + } + + // Sync triggers IPFSPinLs which will return an error + // (see mock) + pi, err := pt.Sync(remoteCid) + if err != nil { + t.Fatal(err) + } + + if pi.Status != api.TrackerStatusRemote || pi.Error != "" { + t.Error("Remote pin should not be in error") + } + + pi = pt.Status(remoteCid) + if err != nil { + t.Fatal(err) + } + + if pi.Status != api.TrackerStatusRemote || pi.Error != "" { + t.Error("Remote pin should not be in error") + } + } + + t.Run("basic pintracker", func(t *testing.T) { + pt := testMapPinTracker(t) + testF(t, pt) + }) + + t.Run("stateless pintracker", func(t *testing.T) { + pt := testStatelessPinTracker(t) + testF(t, pt) + }) +} From d05e8627c5f5fd180b69e0eb3962a19be6764e6b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 23 Oct 2018 19:23:37 +0200 Subject: [PATCH 56/61] Changelog for v0.7.0 License: MIT Signed-off-by: Hector Sanjuan --- CHANGELOG.md | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a17c484..bb716450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,85 @@ # IPFS Cluster Changelog +### v0.7.0 - 2018-10-25 + +#### Summary + +IPFS version 0.7.0 is a maintenance release that includes a few bugfixes and some small features. + +Note that the REST API response format for the `/add` endpoint has changed. Thus all clients need to be upgraded to deal with the new format. The `rest/api/client` has been accordingly updated. + +#### List of changes + +##### Features + + * Clean (rotate) the state when running `init` | [ipfs/ipfs-cluster#532](https://github.com/ipfs/ipfs-cluster/issues/532) | [ipfs/ipfs-cluster#553](https://github.com/ipfs/ipfs-cluster/issues/553) + * Configurable REST API headers and CORS defaults | [ipfs/ipfs-cluster#578](https://github.com/ipfs/ipfs-cluster/issues/578) + * Upgrade libp2p and other deps | [ipfs/ipfs-cluster#580](https://github.com/ipfs/ipfs-cluster/issues/580) | [ipfs/ipfs-cluster#590](https://github.com/ipfs/ipfs-cluster/issues/590) | [ipfs/ipfs-cluster#592](https://github.com/ipfs/ipfs-cluster/issues/592) | [ipfs/ipfs-cluster#598](https://github.com/ipfs/ipfs-cluster/issues/598) | [ipfs/ipfs-cluster#599](https://github.com/ipfs/ipfs-cluster/issues/599) + * Use `gossipsub` to broadcast metrics | [ipfs/ipfs-cluster#573](https://github.com/ipfs/ipfs-cluster/issues/573) + * Download gx and gx-go from IPFS preferentially | [ipfs/ipfs-cluster#577](https://github.com/ipfs/ipfs-cluster/issues/577) | [ipfs/ipfs-cluster#581](https://github.com/ipfs/ipfs-cluster/issues/581) + * Expose peer metrics in the API + ctl commands | [ipfs/ipfs-cluster#449](https://github.com/ipfs/ipfs-cluster/issues/449) | [ipfs/ipfs-cluster#572](https://github.com/ipfs/ipfs-cluster/issues/572) | [ipfs/ipfs-cluster#589](https://github.com/ipfs/ipfs-cluster/issues/589) | [ipfs/ipfs-cluster#587](https://github.com/ipfs/ipfs-cluster/issues/587) + * Add a `docker-compose.yml` template, which creates a two peer cluster | [ipfs/ipfs-cluster#585](https://github.com/ipfs/ipfs-cluster/issues/585) | [ipfs/ipfs-cluster#588](https://github.com/ipfs/ipfs-cluster/issues/588) + * Support overwriting configuration values in the `cluster` section with environmental values | [ipfs/ipfs-cluster#575](https://github.com/ipfs/ipfs-cluster/issues/575) | [ipfs/ipfs-cluster#596](https://github.com/ipfs/ipfs-cluster/issues/596) + * Set snaps to `classic` confinement mode and revert it since approval never arrived | [ipfs/ipfs-cluster#579](https://github.com/ipfs/ipfs-cluster/issues/579) | [ipfs/ipfs-cluster#594](https://github.com/ipfs/ipfs-cluster/issues/594) +* Use Go's reverse proxy library in the proxy endpoint | [ipfs/ipfs-cluster#570](https://github.com/ipfs/ipfs-cluster/issues/570) | [ipfs/ipfs-cluster#605](https://github.com/ipfs/ipfs-cluster/issues/605) + + +##### Bug fixes + + * `/add` endpoints improvements and IPFS Companion compatiblity | [ipfs/ipfs-cluster#582](https://github.com/ipfs/ipfs-cluster/issues/582) | [ipfs/ipfs-cluster#569](https://github.com/ipfs/ipfs-cluster/issues/569) + * Fix adding with spaces in the name parameter | [ipfs/ipfs-cluster#583](https://github.com/ipfs/ipfs-cluster/issues/583) + * Escape filter query parameter | [ipfs/ipfs-cluster#586](https://github.com/ipfs/ipfs-cluster/issues/586) + * Fix some race conditions | [ipfs/ipfs-cluster#597](https://github.com/ipfs/ipfs-cluster/issues/597) + * Improve pin deserialization efficiency | [ipfs/ipfs-cluster#601](https://github.com/ipfs/ipfs-cluster/issues/601) + * Do not error remote pins | [ipfs/ipfs-cluster#600](https://github.com/ipfs/ipfs-cluster/issues/600) | [ipfs/ipfs-cluster#603](https://github.com/ipfs/ipfs-cluster/issues/603) + +#### Upgrading notices + +##### Configuration changes + +The configurations from previous versions are compatible, but a new `headers` key has been added to the `restapi` section. By default it gets CORS headers which will allow read-only interaction from any origin. + +Additionally, all fields from the main `cluster` configuration section can now be overwrriten with environment variables. i.e. `CLUSTER_SECRET`, or `CLUSTER_DISABLEREPINNING`. + +##### REST API + +The `/add` endpoint stream now returns different objects, in line with the rest of the API types. + +Before: + +``` +type AddedOutput struct { + Error + Name string + Hash string `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` +} +``` + +Now: + +``` +type AddedOutput struct { + Name string `json:"name"` + Cid string `json:"cid,omitempty"` + Bytes uint64 `json:"bytes,omitempty"` + Size uint64 `json:"size,omitempty"` +} +``` + +The `/add` endpoint no longer reports errors as part of an AddedOutput object, but instead it uses trailer headers (same as `go-ipfs`). They are handled in the `client`. + +##### Go APIs + +The `AddedOutput` object has changed, thus the `api/rest/client` from older versions will not work with this one. + +##### Other + +No other things. + +--- + ### v0.6.0 - 2018-10-03 #### Summary From 1d924bfc4933a58c425a8efc78027cd639c61c27 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 18:58:46 +0100 Subject: [PATCH 57/61] Remove shard testing folders License: MIT Signed-off-by: Hector Sanjuan --- api/rest/client/methods_test.go | 1 + api/rest/restapi_test.go | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/api/rest/client/methods_test.go b/api/rest/client/methods_test.go index 7c7fa073..0cc14e3c 100644 --- a/api/rest/client/methods_test.go +++ b/api/rest/client/methods_test.go @@ -436,6 +436,7 @@ func TestAddMultiFile(t *testing.T) { testF := func(t *testing.T, c Client) { sth := test.NewShardingTestHelper() + defer sth.Clean(t) mfr, closer := sth.GetTreeMultiReader(t) defer closer.Close() diff --git a/api/rest/restapi_test.go b/api/rest/restapi_test.go index 7de0a3fa..7f0141ee 100644 --- a/api/rest/restapi_test.go +++ b/api/rest/restapi_test.go @@ -377,10 +377,20 @@ func TestAPIAddFileEndpointBadContentType(t *testing.T) { func TestAPIAddFileEndpointLocal(t *testing.T) { rest := testAPI(t) defer rest.Shutdown() + + sth := test.NewShardingTestHelper() + defer sth.Clean(t) + + // This writes generates the testing files and + // writes them to disk. + // This is necessary here because we run tests + // in parallel, and otherwise a write-race might happen. + _, closer := sth.GetTreeMultiReader(t) + closer.Close() + tf := func(t *testing.T, url urlF) { fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1" localURL := url(rest) + fmtStr1 - sth := test.NewShardingTestHelper() body, closer := sth.GetTreeMultiReader(t) defer closer.Close() resp := api.AddedOutput{} @@ -394,8 +404,18 @@ func TestAPIAddFileEndpointLocal(t *testing.T) { func TestAPIAddFileEndpointShard(t *testing.T) { rest := testAPI(t) defer rest.Shutdown() + + sth := test.NewShardingTestHelper() + defer sth.Clean(t) + + // This writes generates the testing files and + // writes them to disk. + // This is necessary here because we run tests + // in parallel, and otherwise a write-race might happen. + _, closer := sth.GetTreeMultiReader(t) + closer.Close() + tf := func(t *testing.T, url urlF) { - sth := test.NewShardingTestHelper() body, closer := sth.GetTreeMultiReader(t) defer closer.Close() mpContentType := "multipart/form-data; boundary=" + body.Boundary() From b47eca40b036e8b1aa6137ec6fd6e5b00ec94090 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 18:59:59 +0100 Subject: [PATCH 58/61] Update changelog License: MIT Signed-off-by: Hector Sanjuan --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb716450..3c0f12a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Note that the REST API response format for the `/add` endpoint has changed. Thus * Fix some race conditions | [ipfs/ipfs-cluster#597](https://github.com/ipfs/ipfs-cluster/issues/597) * Improve pin deserialization efficiency | [ipfs/ipfs-cluster#601](https://github.com/ipfs/ipfs-cluster/issues/601) * Do not error remote pins | [ipfs/ipfs-cluster#600](https://github.com/ipfs/ipfs-cluster/issues/600) | [ipfs/ipfs-cluster#603](https://github.com/ipfs/ipfs-cluster/issues/603) + * Clean up testing folders in `rest` and `rest/client` after tests | [ipfs/ipfs-cluster#607](https://github.com/ipfs/ipfs-cluster/issues/607) #### Upgrading notices From 7e1ddd90a7afa3b1eb35ec3126d4dda83f0be13e Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 19:49:07 +0100 Subject: [PATCH 59/61] Release 0.7.0 License: MIT Signed-off-by: Hector Sanjuan --- cmd/ipfs-cluster-ctl/main.go | 2 +- version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/ipfs-cluster-ctl/main.go b/cmd/ipfs-cluster-ctl/main.go index a8ab6805..56a5bec7 100644 --- a/cmd/ipfs-cluster-ctl/main.go +++ b/cmd/ipfs-cluster-ctl/main.go @@ -27,7 +27,7 @@ const programName = `ipfs-cluster-ctl` // Version is the cluster-ctl tool version. It should match // the IPFS cluster's version -const Version = "0.7.0-rc1" +const Version = "0.7.0" var ( defaultHost = "/ip4/127.0.0.1/tcp/9094" diff --git a/version.go b/version.go index f2348694..5f05f0fc 100644 --- a/version.go +++ b/version.go @@ -9,7 +9,7 @@ import ( // Version is the current cluster version. Version alignment between // components, apis and tools ensures compatibility among them. -var Version = semver.MustParse("0.7.0-rc1") +var Version = semver.MustParse("0.7.0") // RPCProtocol is used to send libp2p messages between cluster peers var RPCProtocol = protocol.ID( From 1b33e2fc9f348f89bfa0f8087658105b5449a442 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 19:49:36 +0100 Subject: [PATCH 60/61] gx publish 0.7.0 License: MIT Signed-off-by: Hector Sanjuan --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index 3fb5af15..6c77ea72 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -0.7.0-rc1: QmUN1SnZ8w8E2AUJzK1VM9ASFdxgHsXEJJXvt7fRcxufnz +0.7.0: QmbGeAqG8wWUqBggxHRCP5ErJGMJHm8SihCfurMmxRU4za diff --git a/package.json b/package.json index cda46510..1b28d97e 100644 --- a/package.json +++ b/package.json @@ -162,6 +162,6 @@ "license": "MIT", "name": "ipfs-cluster", "releaseCmd": "git commit -S -a -m \"gx publish $VERSION\"", - "version": "0.7.0-rc1" + "version": "0.7.0" } From 6f72ab2e67b46f245ed0e8144b065a321d6cb9a9 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Nov 2018 22:55:56 +0100 Subject: [PATCH 61/61] Update 0.7.0 date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0f12a4..1da563db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # IPFS Cluster Changelog -### v0.7.0 - 2018-10-25 +### v0.7.0 - 2018-11-01 #### Summary