Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
package rest
|
2016-12-09 19:54:46 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2018-03-14 22:52:49 +00:00
|
|
|
"context"
|
2018-05-07 12:34:11 +00:00
|
|
|
"crypto/tls"
|
|
|
|
"crypto/x509"
|
2016-12-09 19:54:46 +00:00
|
|
|
"encoding/json"
|
2017-01-30 12:12:25 +00:00
|
|
|
"fmt"
|
2018-04-20 22:57:13 +00:00
|
|
|
"io"
|
2016-12-09 19:54:46 +00:00
|
|
|
"io/ioutil"
|
|
|
|
"net/http"
|
2018-05-04 07:07:12 +00:00
|
|
|
"strings"
|
2018-05-07 12:34:11 +00:00
|
|
|
"testing"
|
2019-01-11 12:25:21 +00:00
|
|
|
"time"
|
2017-02-08 17:04:08 +00:00
|
|
|
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
2017-02-09 15:29:17 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/test"
|
2017-03-10 14:29:11 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
p2phttp "github.com/hsanjuan/go-libp2p-http"
|
|
|
|
libp2p "github.com/libp2p/go-libp2p"
|
|
|
|
host "github.com/libp2p/go-libp2p-host"
|
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
|
|
|
peerstore "github.com/libp2p/go-libp2p-peerstore"
|
2017-03-10 14:29:11 +00:00
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
2016-12-09 19:54:46 +00:00
|
|
|
)
|
|
|
|
|
2018-05-07 12:34:11 +00:00
|
|
|
const (
|
2019-01-11 12:25:21 +00:00
|
|
|
SSLCertFile = "test/server.crt"
|
|
|
|
SSLKeyFile = "test/server.key"
|
|
|
|
clientOrigin = "myorigin"
|
2018-05-04 07:07:12 +00:00
|
|
|
)
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func testAPI(t *testing.T) *API {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-24 18:02:50 +00:00
|
|
|
apiMAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
|
2018-06-27 04:03:15 +00:00
|
|
|
h, err := libp2p.New(ctx, libp2p.ListenAddrs(apiMAddr))
|
2018-03-14 22:52:49 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
|
|
|
|
cfg := &Config{}
|
|
|
|
cfg.Default()
|
2018-03-14 20:59:28 +00:00
|
|
|
cfg.HTTPListenAddr = apiMAddr
|
2019-01-11 12:25:21 +00:00
|
|
|
cfg.CORSAllowedOrigins = []string{clientOrigin}
|
|
|
|
cfg.CORSAllowedMethods = []string{"GET", "POST", "DELETE"}
|
|
|
|
//cfg.CORSAllowedHeaders = []string{"Content-Type"}
|
|
|
|
cfg.CORSMaxAge = 10 * time.Minute
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
rest, err := NewAPIWithHost(ctx, cfg, h)
|
2016-12-09 19:54:46 +00:00
|
|
|
if err != nil {
|
2018-12-14 13:22:03 +00:00
|
|
|
t.Fatal("should be able to create a new API: ", err)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// No keep alive for tests
|
2017-02-08 17:04:08 +00:00
|
|
|
rest.server.SetKeepAlivesEnabled(false)
|
2017-02-09 15:29:17 +00:00
|
|
|
rest.SetClient(test.NewMockRPCClient(t))
|
2018-03-14 22:52:49 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
return rest
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
2018-05-04 07:07:12 +00:00
|
|
|
func testHTTPSAPI(t *testing.T) *API {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-05-04 07:07:12 +00:00
|
|
|
apiMAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
|
2018-06-27 04:03:15 +00:00
|
|
|
h, err := libp2p.New(ctx, libp2p.ListenAddrs(apiMAddr))
|
2018-05-04 07:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
cfg := &Config{}
|
|
|
|
cfg.Default()
|
|
|
|
cfg.pathSSLCertFile = SSLCertFile
|
|
|
|
cfg.pathSSLKeyFile = SSLKeyFile
|
|
|
|
cfg.TLS, err = newTLSConfig(cfg.pathSSLCertFile, cfg.pathSSLKeyFile)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
cfg.HTTPListenAddr = apiMAddr
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
rest, err := NewAPIWithHost(ctx, cfg, h)
|
2018-05-04 07:07:12 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal("should be able to create a new https Api: ", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// No keep alive for tests
|
|
|
|
rest.server.SetKeepAlivesEnabled(false)
|
|
|
|
rest.SetClient(test.NewMockRPCClient(t))
|
|
|
|
|
|
|
|
return rest
|
|
|
|
}
|
|
|
|
|
2016-12-09 19:54:46 +00:00
|
|
|
func processResp(t *testing.T, httpResp *http.Response, err error, resp interface{}) {
|
|
|
|
if err != nil {
|
2018-08-06 20:49:28 +00:00
|
|
|
t.Fatal("error making request: ", err)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
body, err := ioutil.ReadAll(httpResp.Body)
|
|
|
|
defer httpResp.Body.Close()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal("error reading body: ", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(body) != 0 {
|
|
|
|
err = json.Unmarshal(body, resp)
|
|
|
|
if err != nil {
|
|
|
|
t.Error(string(body))
|
|
|
|
t.Fatal("error parsing json: ", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-06 20:49:28 +00:00
|
|
|
func processStreamingResp(t *testing.T, httpResp *http.Response, err error, resp interface{}) {
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal("error making streaming request: ", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if httpResp.StatusCode > 399 {
|
|
|
|
// normal response with error
|
|
|
|
processResp(t, httpResp, err, resp)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
defer httpResp.Body.Close()
|
|
|
|
dec := json.NewDecoder(httpResp.Body)
|
|
|
|
for {
|
|
|
|
err := dec.Decode(&resp)
|
|
|
|
if err == io.EOF {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-16 13:23:06 +00:00
|
|
|
func checkHeaders(t *testing.T, rest *API, url string, headers http.Header) {
|
|
|
|
for k, v := range rest.config.Headers {
|
|
|
|
if strings.Join(v, ",") != strings.Join(headers[k], ",") {
|
|
|
|
t.Errorf("%s does not show configured headers: %s", url, k)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if headers.Get("Content-Type") != "application/json" {
|
|
|
|
t.Errorf("%s is not application/json", url)
|
|
|
|
}
|
2019-01-11 12:25:21 +00:00
|
|
|
|
|
|
|
if eh := headers.Get("Access-Control-Expose-Headers"); eh == "" {
|
|
|
|
t.Error("AC-Expose-Headers not set")
|
|
|
|
}
|
2018-10-16 13:23:06 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// makes a libp2p host that knows how to talk to the rest API host.
|
|
|
|
func makeHost(t *testing.T, rest *API) host.Host {
|
|
|
|
h, err := libp2p.New(context.Background())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
h.Peerstore().AddAddrs(
|
|
|
|
rest.Host().ID(),
|
|
|
|
rest.Host().Addrs(),
|
|
|
|
peerstore.PermanentAddrTTL,
|
|
|
|
)
|
|
|
|
return h
|
|
|
|
}
|
|
|
|
|
|
|
|
type urlF func(a *API) string
|
|
|
|
|
|
|
|
func httpURL(a *API) string {
|
|
|
|
u, _ := a.HTTPAddress()
|
|
|
|
return fmt.Sprintf("http://%s", u)
|
2018-01-24 18:02:50 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
func p2pURL(a *API) string {
|
|
|
|
return fmt.Sprintf("libp2p://%s", peer.IDB58Encode(a.Host().ID()))
|
|
|
|
}
|
|
|
|
|
2018-05-04 07:07:12 +00:00
|
|
|
func httpsURL(a *API) string {
|
|
|
|
u, _ := a.HTTPAddress()
|
|
|
|
return fmt.Sprintf("https://%s", u)
|
|
|
|
}
|
|
|
|
|
2018-07-20 11:56:59 +00:00
|
|
|
func isHTTPS(url string) bool {
|
|
|
|
return strings.HasPrefix(url, "https")
|
|
|
|
}
|
|
|
|
|
2018-05-04 07:07:12 +00:00
|
|
|
// supports both http/https and libp2p-tunneled-http
|
|
|
|
func httpClient(t *testing.T, h host.Host, isHTTPS bool) *http.Client {
|
2018-03-14 22:52:49 +00:00
|
|
|
tr := &http.Transport{}
|
2018-05-04 07:07:12 +00:00
|
|
|
if isHTTPS {
|
|
|
|
certpool := x509.NewCertPool()
|
|
|
|
cert, err := ioutil.ReadFile(SSLCertFile)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal("error reading cert for https client: ", err)
|
|
|
|
}
|
|
|
|
certpool.AppendCertsFromPEM(cert)
|
|
|
|
tr = &http.Transport{
|
|
|
|
TLSClientConfig: &tls.Config{
|
|
|
|
RootCAs: certpool,
|
|
|
|
}}
|
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
if h != nil {
|
|
|
|
tr.RegisterProtocol("libp2p", p2phttp.NewTransport(h))
|
|
|
|
}
|
|
|
|
return &http.Client{Transport: tr}
|
|
|
|
}
|
|
|
|
|
|
|
|
func makeGet(t *testing.T, rest *API, url string, resp interface{}) {
|
|
|
|
h := makeHost(t, rest)
|
|
|
|
defer h.Close()
|
2018-07-20 11:56:59 +00:00
|
|
|
c := httpClient(t, h, isHTTPS(url))
|
2019-01-11 12:25:21 +00:00
|
|
|
req, _ := http.NewRequest(http.MethodGet, url, nil)
|
|
|
|
req.Header.Set("Origin", clientOrigin)
|
|
|
|
httpResp, err := c.Do(req)
|
2016-12-09 19:54:46 +00:00
|
|
|
processResp(t, httpResp, err, resp)
|
2018-10-16 13:23:06 +00:00
|
|
|
checkHeaders(t, rest, url, httpResp.Header)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
func makePost(t *testing.T, rest *API, url string, body []byte, resp interface{}) {
|
2019-01-04 18:31:10 +00:00
|
|
|
makePostWithContentType(t, rest, url, body, "application/json", resp)
|
|
|
|
}
|
|
|
|
|
|
|
|
func makePostWithContentType(t *testing.T, rest *API, url string, body []byte, contentType string, resp interface{}) {
|
2018-03-14 22:52:49 +00:00
|
|
|
h := makeHost(t, rest)
|
|
|
|
defer h.Close()
|
2018-07-20 11:56:59 +00:00
|
|
|
c := httpClient(t, h, isHTTPS(url))
|
2019-01-11 12:25:21 +00:00
|
|
|
req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
|
|
|
req.Header.Set("Content-Type", contentType)
|
|
|
|
req.Header.Set("Origin", clientOrigin)
|
|
|
|
httpResp, err := c.Do(req)
|
2016-12-09 19:54:46 +00:00
|
|
|
processResp(t, httpResp, err, resp)
|
2018-10-16 13:23:06 +00:00
|
|
|
checkHeaders(t, rest, url, httpResp.Header)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
func makeDelete(t *testing.T, rest *API, url string, resp interface{}) {
|
|
|
|
h := makeHost(t, rest)
|
|
|
|
defer h.Close()
|
2018-07-20 11:56:59 +00:00
|
|
|
c := httpClient(t, h, isHTTPS(url))
|
2019-01-11 12:25:21 +00:00
|
|
|
req, _ := http.NewRequest(http.MethodDelete, url, bytes.NewReader([]byte{}))
|
|
|
|
req.Header.Set("Origin", clientOrigin)
|
2016-12-09 19:54:46 +00:00
|
|
|
httpResp, err := c.Do(req)
|
|
|
|
processResp(t, httpResp, err, resp)
|
2018-10-16 13:23:06 +00:00
|
|
|
checkHeaders(t, rest, url, httpResp.Header)
|
2018-04-27 21:12:19 +00:00
|
|
|
}
|
2018-04-20 22:57:13 +00:00
|
|
|
|
2019-01-11 12:25:21 +00:00
|
|
|
func makeOptions(t *testing.T, rest *API, url string, reqHeaders http.Header) http.Header {
|
|
|
|
h := makeHost(t, rest)
|
|
|
|
defer h.Close()
|
|
|
|
c := httpClient(t, h, isHTTPS(url))
|
|
|
|
req, _ := http.NewRequest(http.MethodOptions, url, nil)
|
|
|
|
req.Header = reqHeaders
|
|
|
|
httpResp, err := c.Do(req)
|
|
|
|
processResp(t, httpResp, err, nil)
|
|
|
|
return httpResp.Header
|
|
|
|
}
|
|
|
|
|
2018-08-06 20:49:28 +00:00
|
|
|
func makeStreamingPost(t *testing.T, rest *API, url string, body io.Reader, contentType string, resp interface{}) {
|
2018-04-27 21:12:19 +00:00
|
|
|
h := makeHost(t, rest)
|
|
|
|
defer h.Close()
|
2018-07-20 11:56:59 +00:00
|
|
|
c := httpClient(t, h, isHTTPS(url))
|
2019-01-11 12:25:21 +00:00
|
|
|
req, _ := http.NewRequest(http.MethodPost, url, body)
|
|
|
|
req.Header.Set("Content-Type", contentType)
|
|
|
|
req.Header.Set("Origin", clientOrigin)
|
|
|
|
httpResp, err := c.Do(req)
|
2018-08-06 20:49:28 +00:00
|
|
|
processStreamingResp(t, httpResp, err, resp)
|
2018-10-16 13:23:06 +00:00
|
|
|
checkHeaders(t, rest, url, httpResp.Header)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
type testF func(t *testing.T, url urlF)
|
|
|
|
|
|
|
|
func testBothEndpoints(t *testing.T, test testF) {
|
|
|
|
t.Run("in-parallel", func(t *testing.T) {
|
|
|
|
t.Run("http", func(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
test(t, httpURL)
|
|
|
|
})
|
|
|
|
t.Run("libp2p", func(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
test(t, p2pURL)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-05-04 07:07:12 +00:00
|
|
|
func testHTTPSEndPoint(t *testing.T, test testF) {
|
|
|
|
t.Run("in-parallel", func(t *testing.T) {
|
|
|
|
t.Run("https", func(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
test(t, httpsURL)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIShutdown(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := rest.Shutdown(ctx)
|
2016-12-09 19:54:46 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error("should shutdown cleanly: ", err)
|
|
|
|
}
|
2016-12-23 18:35:37 +00:00
|
|
|
// test shutting down twice
|
2018-06-27 04:03:15 +00:00
|
|
|
rest.Shutdown(ctx)
|
2018-03-14 22:52:49 +00:00
|
|
|
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-24 15:19:23 +00:00
|
|
|
func TestRestAPIIDEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-05-04 07:07:12 +00:00
|
|
|
httpsrest := testHTTPSAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
|
|
|
defer httpsrest.Shutdown(ctx)
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
id := api.IDSerial{}
|
|
|
|
makeGet(t, rest, url(rest)+"/id", &id)
|
|
|
|
if id.ID != test.TestPeerID1.Pretty() {
|
|
|
|
t.Error("expected correct id")
|
|
|
|
}
|
2017-01-24 15:19:23 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
2018-05-04 07:07:12 +00:00
|
|
|
httpstf := func(t *testing.T, url urlF) {
|
|
|
|
id := api.IDSerial{}
|
|
|
|
makeGet(t, httpsrest, url(httpsrest)+"/id", &id)
|
|
|
|
if id.ID != test.TestPeerID1.Pretty() {
|
|
|
|
t.Error("expected correct id")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
testBothEndpoints(t, tf)
|
2018-05-04 07:07:12 +00:00
|
|
|
testHTTPSEndPoint(t, httpstf)
|
2017-01-24 15:19:23 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIVersionEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
ver := api.Version{}
|
|
|
|
makeGet(t, rest, url(rest)+"/version", &ver)
|
|
|
|
if ver.Version != "0.0.mock" {
|
|
|
|
t.Error("expected correct version")
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIPeerstEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var list []api.IDSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/peers", &list)
|
|
|
|
if len(list) != 1 {
|
|
|
|
t.Fatal("expected 1 element")
|
|
|
|
}
|
|
|
|
if list[0].ID != test.TestPeerID1.Pretty() {
|
|
|
|
t.Error("expected a different peer id list: ", list)
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIPeerAddEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
id := api.IDSerial{}
|
|
|
|
// post with valid body
|
2018-07-17 10:51:31 +00:00
|
|
|
body := fmt.Sprintf("{\"peer_id\":\"%s\"}", test.TestPeerID1.Pretty())
|
2018-03-14 22:52:49 +00:00
|
|
|
t.Log(body)
|
|
|
|
makePost(t, rest, url(rest)+"/peers", []byte(body), &id)
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if id.ID != test.TestPeerID1.Pretty() {
|
|
|
|
t.Error("expected correct ID")
|
|
|
|
}
|
|
|
|
if id.Error != "" {
|
|
|
|
t.Error("did not expect an error")
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// Send invalid body
|
|
|
|
errResp := api.Error{}
|
|
|
|
makePost(t, rest, url(rest)+"/peers", []byte("oeoeoeoe"), &errResp)
|
|
|
|
if errResp.Code != 400 {
|
|
|
|
t.Error("expected error with bad body")
|
|
|
|
}
|
2018-07-17 10:51:31 +00:00
|
|
|
// Send invalid peer id
|
|
|
|
makePost(t, rest, url(rest)+"/peers", []byte("{\"peer_id\": \"ab\"}"), &errResp)
|
2018-03-14 22:52:49 +00:00
|
|
|
if errResp.Code != 400 {
|
2018-07-17 10:51:31 +00:00
|
|
|
t.Error("expected error with bad peer_id")
|
2018-03-14 22:52:49 +00:00
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2018-04-27 21:12:19 +00:00
|
|
|
func TestAPIAddFileEndpointBadContentType(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-04-20 22:57:13 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2018-04-20 22:57:13 +00:00
|
|
|
|
2018-04-27 21:12:19 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
2018-08-07 11:42:20 +00:00
|
|
|
fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1"
|
2018-07-20 11:56:59 +00:00
|
|
|
localURL := url(rest) + fmtStr1
|
2018-04-20 22:57:13 +00:00
|
|
|
|
2018-04-27 21:12:19 +00:00
|
|
|
errResp := api.Error{}
|
|
|
|
makePost(t, rest, localURL, []byte("test"), &errResp)
|
|
|
|
|
|
|
|
if errResp.Code != 400 {
|
|
|
|
t.Error("expected error with bad content-type")
|
|
|
|
}
|
2018-04-25 20:35:37 +00:00
|
|
|
}
|
|
|
|
|
2018-04-27 21:12:19 +00:00
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAPIAddFileEndpointLocal(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-04-27 21:12:19 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2018-11-01 17:58:46 +00:00
|
|
|
|
|
|
|
sth := test.NewShardingTestHelper()
|
|
|
|
defer sth.Clean(t)
|
|
|
|
|
2019-01-04 18:31:10 +00:00
|
|
|
// This generates the testing files and
|
2018-11-01 17:58:46 +00:00
|
|
|
// writes them to disk.
|
|
|
|
// This is necessary here because we run tests
|
|
|
|
// in parallel, and otherwise a write-race might happen.
|
|
|
|
_, closer := sth.GetTreeMultiReader(t)
|
|
|
|
closer.Close()
|
|
|
|
|
2018-04-27 21:12:19 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
2019-01-04 18:31:10 +00:00
|
|
|
fmtStr1 := "/add?shard=false&repl_min=-1&repl_max=-1&stream-channels=true"
|
2018-07-20 11:56:59 +00:00
|
|
|
localURL := url(rest) + fmtStr1
|
2018-08-09 08:21:22 +00:00
|
|
|
body, closer := sth.GetTreeMultiReader(t)
|
|
|
|
defer closer.Close()
|
2018-08-06 20:49:28 +00:00
|
|
|
resp := api.AddedOutput{}
|
2018-04-27 21:12:19 +00:00
|
|
|
mpContentType := "multipart/form-data; boundary=" + body.Boundary()
|
2018-08-06 20:49:28 +00:00
|
|
|
makeStreamingPost(t, rest, localURL, body, mpContentType, &resp)
|
2019-01-04 18:31:10 +00:00
|
|
|
|
|
|
|
// resp will contain the last object from the streaming
|
|
|
|
if resp.Cid != test.ShardingDirBalancedRootCID {
|
|
|
|
t.Error("Bad Cid after adding: ", resp.Cid)
|
|
|
|
}
|
2018-04-25 20:35:37 +00:00
|
|
|
}
|
2018-04-27 21:12:19 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAPIAddFileEndpointShard(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-04-27 21:12:19 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2018-11-01 17:58:46 +00:00
|
|
|
|
|
|
|
sth := test.NewShardingTestHelper()
|
|
|
|
defer sth.Clean(t)
|
|
|
|
|
2019-01-04 18:31:10 +00:00
|
|
|
// This generates the testing files and
|
2018-11-01 17:58:46 +00:00
|
|
|
// writes them to disk.
|
|
|
|
// This is necessary here because we run tests
|
|
|
|
// in parallel, and otherwise a write-race might happen.
|
|
|
|
_, closer := sth.GetTreeMultiReader(t)
|
|
|
|
closer.Close()
|
|
|
|
|
2018-04-27 21:12:19 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
2018-08-09 08:21:22 +00:00
|
|
|
body, closer := sth.GetTreeMultiReader(t)
|
|
|
|
defer closer.Close()
|
2018-04-27 21:12:19 +00:00
|
|
|
mpContentType := "multipart/form-data; boundary=" + body.Boundary()
|
2018-08-06 20:49:28 +00:00
|
|
|
resp := api.AddedOutput{}
|
2019-01-04 18:31:10 +00:00
|
|
|
fmtStr1 := "/add?shard=true&repl_min=-1&repl_max=-1&stream-channels=true"
|
2018-07-20 11:56:59 +00:00
|
|
|
shardURL := url(rest) + fmtStr1
|
2018-08-06 20:49:28 +00:00
|
|
|
makeStreamingPost(t, rest, shardURL, body, mpContentType, &resp)
|
2018-04-20 22:57:13 +00:00
|
|
|
}
|
2018-04-27 21:12:19 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2018-04-20 22:57:13 +00:00
|
|
|
}
|
|
|
|
|
2019-01-04 18:31:10 +00:00
|
|
|
func TestAPIAddFileEndpoint_StreamChannelsFalse(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2019-01-04 18:31:10 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2019-01-04 18:31:10 +00:00
|
|
|
|
|
|
|
sth := test.NewShardingTestHelper()
|
|
|
|
defer sth.Clean(t)
|
|
|
|
|
|
|
|
// This generates the testing files and
|
|
|
|
// writes them to disk.
|
|
|
|
// This is necessary here because we run tests
|
|
|
|
// in parallel, and otherwise a write-race might happen.
|
|
|
|
_, closer := sth.GetTreeMultiReader(t)
|
|
|
|
closer.Close()
|
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
body, closer := sth.GetTreeMultiReader(t)
|
|
|
|
defer closer.Close()
|
|
|
|
fullBody, err := ioutil.ReadAll(body)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
mpContentType := "multipart/form-data; boundary=" + body.Boundary()
|
|
|
|
resp := []api.AddedOutput{}
|
|
|
|
fmtStr1 := "/add?shard=false&repl_min=-1&repl_max=-1&stream-channels=false"
|
|
|
|
shardURL := url(rest) + fmtStr1
|
|
|
|
|
|
|
|
makePostWithContentType(t, rest, shardURL, fullBody, mpContentType, &resp)
|
|
|
|
lastHash := resp[len(resp)-1]
|
|
|
|
if lastHash.Cid != test.ShardingDirBalancedRootCID {
|
|
|
|
t.Error("Bad Cid after adding: ", lastHash.Cid)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIPeerRemoveEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
makeDelete(t, rest, url(rest)+"/peers/"+test.TestPeerID1.Pretty(), &struct{}{})
|
|
|
|
}
|
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2018-01-18 02:49:35 +00:00
|
|
|
func TestConnectGraphEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-18 02:49:35 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var cg api.ConnectGraphSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/health/graph", &cg)
|
|
|
|
if cg.ClusterID != test.TestPeerID1.Pretty() {
|
|
|
|
t.Error("unexpected cluster id")
|
|
|
|
}
|
|
|
|
if len(cg.IPFSLinks) != 3 {
|
|
|
|
t.Error("unexpected number of ipfs peers")
|
|
|
|
}
|
|
|
|
if len(cg.ClusterLinks) != 3 {
|
|
|
|
t.Error("unexpected number of cluster peers")
|
|
|
|
}
|
|
|
|
if len(cg.ClustertoIPFS) != 3 {
|
|
|
|
t.Error("unexpected number of cluster to ipfs links")
|
|
|
|
}
|
|
|
|
// test a few link values
|
|
|
|
pid1 := test.TestPeerID1.Pretty()
|
|
|
|
pid4 := test.TestPeerID4.Pretty()
|
|
|
|
if _, ok := cg.ClustertoIPFS[pid1]; !ok {
|
|
|
|
t.Fatal("missing cluster peer 1 from cluster to peer links map")
|
|
|
|
}
|
|
|
|
if cg.ClustertoIPFS[pid1] != pid4 {
|
|
|
|
t.Error("unexpected ipfs peer mapped to cluster peer 1 in graph")
|
|
|
|
}
|
2018-01-18 02:49:35 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2018-01-18 02:49:35 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIPinEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
// test regular post
|
|
|
|
makePost(t, rest, url(rest)+"/pins/"+test.TestCid1, []byte{}, &struct{}{})
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
errResp := api.Error{}
|
|
|
|
makePost(t, rest, url(rest)+"/pins/"+test.ErrorCid, []byte{}, &errResp)
|
|
|
|
if errResp.Message != test.ErrBadCid.Error() {
|
|
|
|
t.Error("expected different error: ", errResp.Message)
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
makePost(t, rest, url(rest)+"/pins/abcd", []byte{}, &errResp)
|
|
|
|
if errResp.Code != 400 {
|
|
|
|
t.Error("should fail with bad Cid")
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
type pathCase struct {
|
|
|
|
path string
|
|
|
|
opts api.PinOptions
|
|
|
|
wantErr bool
|
|
|
|
code int
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *pathCase) WithQuery() string {
|
|
|
|
return p.path + "?" + p.opts.ToQuery()
|
|
|
|
}
|
|
|
|
|
|
|
|
var testPinOpts = api.PinOptions{
|
|
|
|
ReplicationFactorMax: 7,
|
|
|
|
ReplicationFactorMin: 6,
|
|
|
|
Name: "hello there",
|
|
|
|
}
|
|
|
|
|
|
|
|
var pathTestCases = []pathCase{
|
|
|
|
{
|
|
|
|
"/ipfs/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY",
|
|
|
|
testPinOpts,
|
|
|
|
false,
|
|
|
|
http.StatusOK,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"/ipfs/QmbUNM297ZwxB8CfFAznK7H9YMesDoY6Tt5bPgt5MSCB2u/im.gif",
|
|
|
|
testPinOpts,
|
|
|
|
false,
|
|
|
|
http.StatusOK,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"/ipfs/invalidhash",
|
|
|
|
testPinOpts,
|
|
|
|
true,
|
|
|
|
http.StatusBadRequest,
|
|
|
|
},
|
|
|
|
// TODO: Test StatusNotFound and a case with trailing slash with paths
|
|
|
|
// test.TestPathIPNS2, test.TestPathIPLD2, test.TestInvalidPath1
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAPIPinEndpointWithPath(t *testing.T) {
|
|
|
|
ctx := context.Background()
|
|
|
|
rest := testAPI(t)
|
|
|
|
defer rest.Shutdown(ctx)
|
|
|
|
|
|
|
|
resultantPin := api.PinWithOpts(
|
|
|
|
test.MustDecodeCid(test.TestCidResolved),
|
|
|
|
testPinOpts,
|
|
|
|
)
|
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
for _, testCase := range pathTestCases {
|
|
|
|
if testCase.wantErr {
|
|
|
|
errResp := api.Error{}
|
|
|
|
makePost(t, rest, url(rest)+"/pins"+testCase.WithQuery(), []byte{}, &errResp)
|
|
|
|
if errResp.Code != testCase.code {
|
|
|
|
t.Errorf("expected different status code, expected: %d, actual: %d, path: %s\n", testCase.code, errResp.Code, testCase.path)
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
pin := api.PinSerial{}
|
|
|
|
makePost(t, rest, url(rest)+"/pins"+testCase.WithQuery(), []byte{}, &pin)
|
|
|
|
if !pin.ToPin().Equals(resultantPin) {
|
|
|
|
t.Errorf("expected different pin,\n expected: %+v,\n actual: %+v,\n path: %s\n", resultantPin.ToSerial(), pin, testCase.path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIUnpinEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
// test regular delete
|
|
|
|
makeDelete(t, rest, url(rest)+"/pins/"+test.TestCid1, &struct{}{})
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
errResp := api.Error{}
|
|
|
|
makeDelete(t, rest, url(rest)+"/pins/"+test.ErrorCid, &errResp)
|
|
|
|
if errResp.Message != test.ErrBadCid.Error() {
|
|
|
|
t.Error("expected different error: ", errResp.Message)
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
makeDelete(t, rest, url(rest)+"/pins/abcd", &errResp)
|
|
|
|
if errResp.Code != 400 {
|
|
|
|
t.Error("should fail with bad Cid")
|
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
|
|
|
|
Add PinPath/UnpinPath support.
Squashed commit of the following:
commit 38cf569c6aed77c46ee4e0f8baa4d1a9daf8f03e
Merge: d125f69 aaada42
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 20 11:02:00 2019 +0000
Merge pull request #634 from ipfs/issue_450
Support PinPath, UnpinPath (resolve before pinning)
commit aaada42054e1f1c7b2abb1270859d0de41a0e5d8
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 22:16:25 2019 +0530
formatResponse accepts api.Pin and not api.PinSerial
commit b5da4bea045865814cc422da71827b44ddd44b90
Merge: ba59036 cc8dd7e
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 19 21:36:46 2019 +0530
Merge branch 'master' into issue_450
commit ba5903649c1df1dba20f4d6f7e3573d6fe24921f
Merge: f002914 d59880c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:41:11 2019 +0530
Merge branch 'issue_450' of github.com:ipfs/ipfs-cluster into issue_450
commit f00291494c0c02621c2296cbb7ac71e4c23aa9ec
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Feb 18 08:31:39 2019 +0530
PinPath: more improvements
Added tracing for new methods
commit d59880c338eaa8214fe06b4f930a540793d78407
Merge: 0ca4c7c b4f0eb3
Author: Hector Sanjuan <hsanjuan@users.noreply.github.com>
Date: Wed Feb 13 15:22:49 2019 +0000
Merge branch 'master' into issue_450
commit 0ca4c7c3b0670ed9c8279f8274d36e3485c10030
Merge: d35017a ecef9ea
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:10:13 2019 +0530
Merge branch 'master' into issue_450
commit d35017a8de91ca9fc9a9a047c48c75134cee9f98
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 12 13:07:25 2019 +0530
PinPath: more improvements
- Worth having `PinOptions` as a separate field in the struct and
constructing the query in the test with ToQuery()
- sharness: "intialization" line can be placed outside the tests at
the top
commit 68e3b90417ffbad89d41a70ac81d85f9037f8848
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Sun Feb 10 21:43:50 2019 +0530
Using if-continue pattern instead of if-else
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 3c29799f3b85be328b27508332ab92049d8b82f3
Merge: 956790b 4324889
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Feb 7 10:25:52 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 956790b381db9858e4194f983e898b07dc51ba66
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 21:11:20 2019 +0530
Removing resolved path
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7191cc46cedfbec116a9746937e28881b50ca044
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:45:07 2019 +0530
Fix go vet
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f8b3d5b63b1b7569e2a3e0d82894fd4491c246c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 16:07:03 2019 +0530
Fixed linting error
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 23c57eb467755a1f21387a1615a7f34e97348053
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 09:20:41 2019 +0530
Fixed tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 0caedd94aefeb3b6649dedc214cb4b849ace2ea4
Merge: 17e555e 5a7ee1d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Feb 6 00:07:10 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 17e555e4a7c574413df90aac70c5cc29cab98f54
Author: Hector Sanjuan <code@hector.link>
Date: Tue Feb 5 16:58:50 2019 +0000
PinPath: address some feedback + improvements
* Changed client's Pin() API and PinPath to be consistent
* Added helper methods to turn PinPath to query and back
* Make code and tests build
* Use TestCidResolved everywhere
* Fix cluster.PinPath arguments
* Fix formatting of responses with --no-status
* Make tests readable and call Fatal when needed
* Use a pathTestCases variable
commit f0e7369c47c5ddadc8ed45df5fd2d4d9b2d42b38
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Feb 5 18:34:26 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Addressed review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-198751932
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit a8b4f181d2d7afed32ee41331dfaab19fd66a173
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 22:41:27 2019 +0530
Fixing tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit e39b95ca19e4d75506f4f492678245ef13936a44
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 29 14:52:53 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- PinPath and UnpinPath should return api.Pin
- PinPath should accept pin options
- Removing duplicate logic for Resolve from cluster
- And many other review comments https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-195509504
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit d146075126320896665ba58d337a13789f68ea86
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 17:08:41 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
PinPath(in both rest and rpc) should return a serializable struct in the
form `{"\":"Q...cid..string..."}` (as used in "github.com/ipfs/go-cid"
to marshal and unmarshal)
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 1f4869568a8adb450275257154ea3a26d03a30f3
Merge: 7acfd28 a244af9
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 23 07:18:56 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 7acfd282732ddf2282a67d4f9d0170a494eb3ed4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 22 18:14:32 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- RPC must always use serializable structs
- In command, just use pin with path as cid is also a valid path
- Addressing many other small review comments as in
https://github.com/ipfs/ipfs-cluster/pull/634#pullrequestreview-192122534
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 36905041e1e3f0b204942030aab3ab7b5b9e4d62
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 09:36:42 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Extra logic for path checking should go into resolve so that it can be
properly reused
Added sharness tests
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 9116bda3534e77bb391d873051bb520a1b01a326
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 16 08:08:07 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
error strings should not be capitalized
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit ca7e61861374f456300a85ddc0374e594f74f963
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 23:40:25 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
Tests
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 522fbcd899f01c01680375561a32a87464157c0a
Merge: f1a56ab f7bc468
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 15 10:40:54 2019 +0530
Merge branch 'master' into issue_450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit f1a56ab925fb74c0c44273a4524afa4843cf757f
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Mon Jan 14 20:58:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- IPFS Connector should act as a pure IPFS client, any extra logic
should go to cluster.go
- Use cid.Undef, instead of cid.Cid{}
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit c83b91054f6774f1f9d4930cfc3f1fa28236f57c
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Thu Jan 10 08:57:17 2019 +0530
Support PinPath, UnpinPath(resolve before pinning)
- Separate handlers, methods and rpc apis for PinPath and UnpinPath from
Pin and Unpin
- Support ipld paths as well
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 719dff88129366ce3ccb5e04cb6f8082a0915c5c
Merge: 91ceb47 21170c4
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:38:35 2019 +0530
Merge branch 'issue_450_old' into HEAD
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 91ceb4796259ca7ef2974ec43e6a278a12796b13
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Wed Jan 9 19:36:41 2019 +0530
Revert "WIP: Figure out why test does not impleme"
This reverts commit 28a3a3f25dce6f296c8cbef86221644c099a7e75.
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
commit 28a3a3f25dce6f296c8cbef86221644c099a7e75
Author: cd10012 <ced361@nyu.edu>
Date: Tue Jul 24 23:23:10 2018 -0400
WIP: Figure out why test does not implement IPFSConnector interface...
License: MIT
Signed-off-by: cd10012 <ced361@nyu.edu>
commit 21170c48e77e69583db64544b08120a9baf40d8d
Author: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Date: Tue Jan 8 10:37:59 2019 +0530
Support PinPath, UnpinPath (resolve before pinning)
This commit adds API support for pinning using path
`POST /pins/<ipfs or ipns path>` and `DELETE /pins/<ipfs or ipns path>`
will resolve the path into a cid and perform perform pinning or
unpinning
Fixes #450
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
Co-authored-by: Hector Sanjuan <hector@protocol.ai>
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2019-02-20 11:07:50 +00:00
|
|
|
func TestAPIUnpinEndpointWithPath(t *testing.T) {
|
|
|
|
ctx := context.Background()
|
|
|
|
rest := testAPI(t)
|
|
|
|
defer rest.Shutdown(ctx)
|
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
for _, testCase := range pathTestCases {
|
|
|
|
if testCase.wantErr {
|
|
|
|
errResp := api.Error{}
|
|
|
|
makeDelete(t, rest, url(rest)+"/pins"+testCase.path, &errResp)
|
|
|
|
if errResp.Code != testCase.code {
|
|
|
|
t.Errorf("expected different status code, expected: %d, actual: %d, path: %s\n", testCase.code, errResp.Code, testCase.path)
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
pin := api.PinSerial{}
|
|
|
|
makeDelete(t, rest, url(rest)+"/pins"+testCase.path, &pin)
|
|
|
|
if pin.Cid != test.TestCidResolved {
|
|
|
|
t.Errorf("expected different cid, expected: %s, actual: %s, path: %s\n", test.TestCidResolved, pin.Cid, testCase.path)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIAllocationsEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-15 18:08:46 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp []api.PinSerial
|
2018-08-10 12:39:44 +00:00
|
|
|
makeGet(t, rest, url(rest)+"/allocations?filter=pin,meta-pin", &resp)
|
2018-03-14 22:52:49 +00:00
|
|
|
if len(resp) != 3 ||
|
|
|
|
resp[0].Cid != test.TestCid1 || resp[1].Cid != test.TestCid2 ||
|
|
|
|
resp[2].Cid != test.TestCid3 {
|
|
|
|
t.Error("unexpected pin list: ", resp)
|
|
|
|
}
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIAllocationEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2017-04-06 02:27:02 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp api.PinSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/allocations/"+test.TestCid1, &resp)
|
|
|
|
if resp.Cid != test.TestCid1 {
|
|
|
|
t.Error("cid should be the same")
|
|
|
|
}
|
2017-04-06 02:27:02 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
errResp := api.Error{}
|
|
|
|
makeGet(t, rest, url(rest)+"/allocations/"+test.ErrorCid, &errResp)
|
|
|
|
if errResp.Code != 404 {
|
|
|
|
t.Error("a non-pinned cid should 404")
|
|
|
|
}
|
2017-04-06 02:27:02 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2017-04-06 02:27:02 +00:00
|
|
|
}
|
|
|
|
|
2018-12-14 13:22:03 +00:00
|
|
|
func TestAPIMetricsEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-12-14 13:22:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2018-12-14 13:22:03 +00:00
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
2018-12-21 02:47:55 +00:00
|
|
|
var resp []api.MetricSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/monitor/metrics/somemetricstype", &resp)
|
|
|
|
if len(resp) == 0 {
|
|
|
|
t.Fatal("No metrics found")
|
|
|
|
}
|
|
|
|
for _, m := range resp {
|
|
|
|
if m.Name != "test" {
|
|
|
|
t.Error("Unexpected metric name: ", m.Name)
|
2018-12-14 13:22:03 +00:00
|
|
|
}
|
2018-12-21 02:47:55 +00:00
|
|
|
if m.Peer != test.TestPeerID1.Pretty() {
|
|
|
|
t.Error("Unexpected peer id: ", m.Peer)
|
2018-12-14 13:22:03 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|
2018-12-17 14:52:43 +00:00
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIStatusAllEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp []api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins", &resp)
|
|
|
|
if len(resp) != 3 ||
|
|
|
|
resp[0].Cid != test.TestCid1 ||
|
|
|
|
resp[1].PeerMap[test.TestPeerID1.Pretty()].Status != "pinning" {
|
|
|
|
t.Errorf("unexpected statusAll resp:\n %+v", resp)
|
|
|
|
}
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// Test local=true
|
|
|
|
var resp2 []api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins?local=true", &resp2)
|
|
|
|
if len(resp2) != 2 {
|
Status filters for `ipfs-cluster-ctl status`
Added filter option to `ipfs-cluster-ctl status`
When the --filter is passed, it will only fetch the peer information
where status of the pin matches with the filter value.
Valid filter values are tracker status types(i.e., "pinned",
"pin_error", "unpinning" etc), an alias of tracker status type (i.e.,
"queued" or "error"), comma separated list of tracker status type
and/or it aliases(i.e., "error,pinning")
On passing invalid filter value no status information will be shown
In particular, the filter would remove elements from []GlobalPinInfo
when none of the peers in GlobalPinInfo match the filter. If one peer
in the GlobalPinInfo matches the filter, the whole object is returned,
including the information for the other peers which may or not match it.
filter option works on statusAll("GET /pins"). For fetching pin status
for a CID("GET /pins/<cid>"), filter option would have no effect
Fixes #445
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
2018-12-20 08:02:26 +00:00
|
|
|
t.Errorf("unexpected statusAll+local resp:\n %+v", resp2)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test with filter
|
|
|
|
var resp3 []api.GlobalPinInfoSerial
|
2018-12-26 05:45:30 +00:00
|
|
|
makeGet(t, rest, url(rest)+"/pins?filter=queued", &resp3)
|
Status filters for `ipfs-cluster-ctl status`
Added filter option to `ipfs-cluster-ctl status`
When the --filter is passed, it will only fetch the peer information
where status of the pin matches with the filter value.
Valid filter values are tracker status types(i.e., "pinned",
"pin_error", "unpinning" etc), an alias of tracker status type (i.e.,
"queued" or "error"), comma separated list of tracker status type
and/or it aliases(i.e., "error,pinning")
On passing invalid filter value no status information will be shown
In particular, the filter would remove elements from []GlobalPinInfo
when none of the peers in GlobalPinInfo match the filter. If one peer
in the GlobalPinInfo matches the filter, the whole object is returned,
including the information for the other peers which may or not match it.
filter option works on statusAll("GET /pins"). For fetching pin status
for a CID("GET /pins/<cid>"), filter option would have no effect
Fixes #445
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
2018-12-20 08:02:26 +00:00
|
|
|
if len(resp3) != 0 {
|
2018-12-26 05:45:30 +00:00
|
|
|
t.Errorf("unexpected statusAll+filter=queued resp:\n %+v", resp3)
|
Status filters for `ipfs-cluster-ctl status`
Added filter option to `ipfs-cluster-ctl status`
When the --filter is passed, it will only fetch the peer information
where status of the pin matches with the filter value.
Valid filter values are tracker status types(i.e., "pinned",
"pin_error", "unpinning" etc), an alias of tracker status type (i.e.,
"queued" or "error"), comma separated list of tracker status type
and/or it aliases(i.e., "error,pinning")
On passing invalid filter value no status information will be shown
In particular, the filter would remove elements from []GlobalPinInfo
when none of the peers in GlobalPinInfo match the filter. If one peer
in the GlobalPinInfo matches the filter, the whole object is returned,
including the information for the other peers which may or not match it.
filter option works on statusAll("GET /pins"). For fetching pin status
for a CID("GET /pins/<cid>"), filter option would have no effect
Fixes #445
License: MIT
Signed-off-by: Kishan Mohanbhai Sagathiya <kishansagathiya@gmail.com>
2018-12-20 08:02:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var resp4 []api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins?filter=pinned", &resp4)
|
|
|
|
if len(resp4) != 1 {
|
|
|
|
t.Errorf("unexpected statusAll+filter=pinned resp:\n %+v", resp4)
|
|
|
|
}
|
|
|
|
|
|
|
|
var resp5 []api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins?filter=pin_error", &resp5)
|
|
|
|
if len(resp5) != 1 {
|
|
|
|
t.Errorf("unexpected statusAll+filter=pin_error resp:\n %+v", resp5)
|
|
|
|
}
|
|
|
|
|
|
|
|
var resp6 []api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins?filter=error", &resp6)
|
|
|
|
if len(resp6) != 1 {
|
|
|
|
t.Errorf("unexpected statusAll+filter=error resp:\n %+v", resp6)
|
|
|
|
}
|
|
|
|
|
|
|
|
var resp7 []api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins?filter=error,pinned", &resp7)
|
|
|
|
if len(resp7) != 2 {
|
|
|
|
t.Errorf("unexpected statusAll+filter=error,pinned resp:\n %+v", resp7)
|
2018-03-14 22:52:49 +00:00
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIStatusEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins/"+test.TestCid1, &resp)
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if resp.Cid != test.TestCid1 {
|
|
|
|
t.Error("expected the same cid")
|
|
|
|
}
|
|
|
|
info, ok := resp.PeerMap[test.TestPeerID1.Pretty()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("expected info for test.TestPeerID1")
|
|
|
|
}
|
|
|
|
if info.Status != "pinned" {
|
|
|
|
t.Error("expected different status")
|
|
|
|
}
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// Test local=true
|
|
|
|
var resp2 api.GlobalPinInfoSerial
|
|
|
|
makeGet(t, rest, url(rest)+"/pins/"+test.TestCid1+"?local=true", &resp2)
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if resp2.Cid != test.TestCid1 {
|
|
|
|
t.Error("expected the same cid")
|
|
|
|
}
|
|
|
|
info, ok = resp2.PeerMap[test.TestPeerID2.Pretty()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("expected info for test.TestPeerID2")
|
|
|
|
}
|
|
|
|
if info.Status != "pinned" {
|
|
|
|
t.Error("expected different status")
|
|
|
|
}
|
2017-12-01 11:56:26 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPISyncAllEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp []api.GlobalPinInfoSerial
|
|
|
|
makePost(t, rest, url(rest)+"/pins/sync", []byte{}, &resp)
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if len(resp) != 3 ||
|
|
|
|
resp[0].Cid != test.TestCid1 ||
|
|
|
|
resp[1].PeerMap[test.TestPeerID1.Pretty()].Status != "pinning" {
|
|
|
|
t.Errorf("unexpected syncAll resp:\n %+v", resp)
|
|
|
|
}
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// Test local=true
|
|
|
|
var resp2 []api.GlobalPinInfoSerial
|
|
|
|
makePost(t, rest, url(rest)+"/pins/sync?local=true", []byte{}, &resp2)
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if len(resp2) != 2 {
|
|
|
|
t.Errorf("unexpected syncAll+local resp:\n %+v", resp2)
|
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPISyncEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2017-01-25 18:38:23 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp api.GlobalPinInfoSerial
|
|
|
|
makePost(t, rest, url(rest)+"/pins/"+test.TestCid1+"/sync", []byte{}, &resp)
|
2017-01-25 18:38:23 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if resp.Cid != test.TestCid1 {
|
|
|
|
t.Error("expected the same cid")
|
|
|
|
}
|
|
|
|
info, ok := resp.PeerMap[test.TestPeerID1.Pretty()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("expected info for test.TestPeerID1")
|
|
|
|
}
|
|
|
|
if info.Status != "pinned" {
|
|
|
|
t.Error("expected different status")
|
|
|
|
}
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
// Test local=true
|
|
|
|
var resp2 api.GlobalPinInfoSerial
|
|
|
|
makePost(t, rest, url(rest)+"/pins/"+test.TestCid1+"/sync?local=true", []byte{}, &resp2)
|
2017-12-01 11:56:26 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if resp2.Cid != test.TestCid1 {
|
|
|
|
t.Error("expected the same cid")
|
|
|
|
}
|
|
|
|
info, ok = resp2.PeerMap[test.TestPeerID2.Pretty()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("expected info for test.TestPeerID2")
|
|
|
|
}
|
|
|
|
if info.Status != "pinned" {
|
|
|
|
t.Error("expected different status")
|
|
|
|
}
|
2017-12-01 11:56:26 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2017-01-25 18:38:23 +00:00
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func TestAPIRecoverEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp api.GlobalPinInfoSerial
|
|
|
|
makePost(t, rest, url(rest)+"/pins/"+test.TestCid1+"/recover", []byte{}, &resp)
|
2016-12-23 18:35:37 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if resp.Cid != test.TestCid1 {
|
|
|
|
t.Error("expected the same cid")
|
|
|
|
}
|
|
|
|
info, ok := resp.PeerMap[test.TestPeerID1.Pretty()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("expected info for test.TestPeerID1")
|
|
|
|
}
|
|
|
|
if info.Status != "pinned" {
|
|
|
|
t.Error("expected different status")
|
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2017-11-30 00:53:31 +00:00
|
|
|
|
|
|
|
func TestAPIRecoverAllEndpoint(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-11-30 00:53:31 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2017-11-30 00:53:31 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
var resp []api.GlobalPinInfoSerial
|
|
|
|
makePost(t, rest, url(rest)+"/pins/recover?local=true", []byte{}, &resp)
|
2017-11-30 00:53:31 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
if len(resp) != 0 {
|
|
|
|
t.Fatal("bad response length")
|
|
|
|
}
|
2017-11-30 00:53:31 +00:00
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
var errResp api.Error
|
|
|
|
makePost(t, rest, url(rest)+"/pins/recover", []byte{}, &errResp)
|
|
|
|
if errResp.Code != 400 {
|
|
|
|
t.Error("expected a different error")
|
|
|
|
}
|
2017-11-30 00:53:31 +00:00
|
|
|
}
|
2018-03-14 22:52:49 +00:00
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
2017-11-30 00:53:31 +00:00
|
|
|
}
|
2019-01-11 12:25:21 +00:00
|
|
|
|
|
|
|
func TestCORS(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2019-01-11 12:25:21 +00:00
|
|
|
rest := testAPI(t)
|
2018-06-27 04:03:15 +00:00
|
|
|
defer rest.Shutdown(ctx)
|
2019-01-11 12:25:21 +00:00
|
|
|
|
|
|
|
type testcase struct {
|
|
|
|
method string
|
|
|
|
path string
|
|
|
|
}
|
|
|
|
|
|
|
|
tf := func(t *testing.T, url urlF) {
|
|
|
|
reqHeaders := make(http.Header)
|
|
|
|
reqHeaders.Set("Origin", "myorigin")
|
|
|
|
reqHeaders.Set("Access-Control-Request-Headers", "Content-Type")
|
|
|
|
|
|
|
|
for _, tc := range []testcase{
|
|
|
|
testcase{"GET", "/pins"},
|
|
|
|
// testcase{},
|
|
|
|
} {
|
|
|
|
reqHeaders.Set("Access-Control-Request-Method", tc.method)
|
|
|
|
headers := makeOptions(t, rest, url(rest)+tc.path, reqHeaders)
|
|
|
|
aorigin := headers.Get("Access-Control-Allow-Origin")
|
|
|
|
amethods := headers.Get("Access-Control-Allow-Methods")
|
|
|
|
aheaders := headers.Get("Access-Control-Allow-Headers")
|
|
|
|
acreds := headers.Get("Access-Control-Allow-Credentials")
|
|
|
|
maxage := headers.Get("Access-Control-Max-Age")
|
|
|
|
|
|
|
|
if aorigin != "myorigin" {
|
|
|
|
t.Error("Bad ACA-Origin:", aorigin)
|
|
|
|
}
|
|
|
|
|
|
|
|
if amethods != tc.method {
|
|
|
|
t.Error("Bad ACA-Methods:", amethods)
|
|
|
|
}
|
|
|
|
|
|
|
|
if aheaders != "Content-Type" {
|
|
|
|
t.Error("Bad ACA-Headers:", aheaders)
|
|
|
|
}
|
|
|
|
|
|
|
|
if acreds != "true" {
|
|
|
|
t.Error("Bad ACA-Credentials:", acreds)
|
|
|
|
}
|
|
|
|
|
|
|
|
if maxage != "600" {
|
|
|
|
t.Error("Bad AC-Max-Age:", maxage)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
testBothEndpoints(t, tf)
|
|
|
|
}
|