2016-12-02 18:33:39 +00:00
|
|
|
package ipfscluster
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2017-02-13 15:46:53 +00:00
|
|
|
"errors"
|
2017-01-30 12:12:25 +00:00
|
|
|
"fmt"
|
2017-02-27 16:28:41 +00:00
|
|
|
"os"
|
|
|
|
"path/filepath"
|
2017-06-30 00:40:06 +00:00
|
|
|
"strings"
|
2016-12-15 13:07:19 +00:00
|
|
|
"sync"
|
2017-01-23 19:29:05 +00:00
|
|
|
"time"
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2017-07-20 11:18:46 +00:00
|
|
|
pnet "github.com/libp2p/go-libp2p-pnet"
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
2017-03-10 16:24:25 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/consensus/raft"
|
|
|
|
"github.com/ipfs/ipfs-cluster/state"
|
2017-02-08 17:04:08 +00:00
|
|
|
|
2017-01-25 11:14:39 +00:00
|
|
|
rpc "github.com/hsanjuan/go-libp2p-gorpc"
|
2016-12-16 21:00:08 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
2016-12-16 11:40:28 +00:00
|
|
|
host "github.com/libp2p/go-libp2p-host"
|
2017-06-30 00:40:06 +00:00
|
|
|
ipnet "github.com/libp2p/go-libp2p-interface-pnet"
|
2016-12-16 11:40:28 +00:00
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
|
|
|
peerstore "github.com/libp2p/go-libp2p-peerstore"
|
|
|
|
swarm "github.com/libp2p/go-libp2p-swarm"
|
|
|
|
basichost "github.com/libp2p/go-libp2p/p2p/host/basic"
|
2017-01-24 15:19:23 +00:00
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
2016-12-02 18:33:39 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
// Cluster is the main IPFS cluster component. It provides
|
2017-01-24 11:39:08 +00:00
|
|
|
// the go-API for it and orchestrates the components that make up the system.
|
2016-12-02 18:33:39 +00:00
|
|
|
type Cluster struct {
|
2017-02-09 15:29:17 +00:00
|
|
|
ctx context.Context
|
|
|
|
cancel func()
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
id peer.ID
|
2017-01-30 12:12:25 +00:00
|
|
|
config *Config
|
|
|
|
host host.Host
|
|
|
|
rpcServer *rpc.Server
|
|
|
|
rpcClient *rpc.Client
|
|
|
|
peerManager *peerManager
|
2016-12-02 18:33:39 +00:00
|
|
|
|
2017-03-14 15:37:29 +00:00
|
|
|
consensus Consensus
|
2016-12-15 18:08:46 +00:00
|
|
|
api API
|
2016-12-02 18:33:39 +00:00
|
|
|
ipfs IPFSConnector
|
2017-03-10 16:24:25 +00:00
|
|
|
state state.State
|
2016-12-06 21:29:59 +00:00
|
|
|
tracker PinTracker
|
2017-02-13 15:46:53 +00:00
|
|
|
monitor PeerMonitor
|
|
|
|
allocator PinAllocator
|
|
|
|
informer Informer
|
2016-12-15 13:07:19 +00:00
|
|
|
|
|
|
|
shutdownLock sync.Mutex
|
2017-10-31 10:20:14 +00:00
|
|
|
shutdownB bool
|
|
|
|
shutdownOnce sync.Once
|
2017-01-30 12:12:25 +00:00
|
|
|
doneCh chan struct{}
|
|
|
|
readyCh chan struct{}
|
2017-10-30 11:17:39 +00:00
|
|
|
readyB bool
|
2016-12-15 13:07:19 +00:00
|
|
|
wg sync.WaitGroup
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
paMux sync.Mutex
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// NewCluster builds a new IPFS Cluster peer. It initializes a LibP2P host,
|
|
|
|
// creates and RPC Server and client and sets up all components.
|
|
|
|
//
|
|
|
|
// The new cluster peer may still be performing initialization tasks when
|
|
|
|
// this call returns (consensus may still be bootstrapping). Use Cluster.Ready()
|
|
|
|
// if you need to wait until the peer is fully up.
|
2017-02-13 15:46:53 +00:00
|
|
|
func NewCluster(
|
|
|
|
cfg *Config,
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
consensusCfg *raft.Config,
|
2017-02-13 15:46:53 +00:00
|
|
|
api API,
|
|
|
|
ipfs IPFSConnector,
|
2017-03-10 16:24:25 +00:00
|
|
|
st state.State,
|
2017-02-13 15:46:53 +00:00
|
|
|
tracker PinTracker,
|
|
|
|
monitor PeerMonitor,
|
|
|
|
allocator PinAllocator,
|
|
|
|
informer Informer) (*Cluster, error) {
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
err := cfg.Validate()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2016-12-02 18:33:39 +00:00
|
|
|
host, err := makeHost(ctx, cfg)
|
|
|
|
if err != nil {
|
2017-02-15 14:16:34 +00:00
|
|
|
cancel()
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
logger.Infof("IPFS Cluster v%s listening on:", Version)
|
|
|
|
for _, addr := range host.Addrs() {
|
|
|
|
logger.Infof(" %s/ipfs/%s", addr, host.ID().Pretty())
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
c := &Cluster{
|
2017-02-13 15:46:53 +00:00
|
|
|
ctx: ctx,
|
|
|
|
cancel: cancel,
|
|
|
|
id: host.ID(),
|
|
|
|
config: cfg,
|
|
|
|
host: host,
|
|
|
|
api: api,
|
|
|
|
ipfs: ipfs,
|
2017-03-10 16:24:25 +00:00
|
|
|
state: st,
|
2017-02-13 15:46:53 +00:00
|
|
|
tracker: tracker,
|
|
|
|
monitor: monitor,
|
|
|
|
allocator: allocator,
|
|
|
|
informer: informer,
|
2017-10-31 10:20:14 +00:00
|
|
|
shutdownB: false,
|
2017-02-13 15:46:53 +00:00
|
|
|
doneCh: make(chan struct{}),
|
|
|
|
readyCh: make(chan struct{}),
|
2017-10-30 11:17:39 +00:00
|
|
|
readyB: false,
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
c.setupPeerManager()
|
|
|
|
err = c.setupRPC()
|
2016-12-23 18:35:37 +00:00
|
|
|
if err != nil {
|
2017-02-02 22:52:06 +00:00
|
|
|
c.Shutdown()
|
2016-12-23 18:35:37 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
err = c.setupConsensus(consensusCfg)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
2017-02-02 22:52:06 +00:00
|
|
|
c.Shutdown()
|
2017-01-30 12:12:25 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.setupRPCClients()
|
2017-02-13 15:46:53 +00:00
|
|
|
ok := c.bootstrap()
|
|
|
|
if !ok {
|
|
|
|
logger.Error("Bootstrap unsuccessful")
|
|
|
|
c.Shutdown()
|
|
|
|
return nil, errors.New("bootstrap unsuccessful")
|
|
|
|
}
|
|
|
|
go func() {
|
|
|
|
c.ready()
|
|
|
|
c.run()
|
|
|
|
}()
|
2017-02-02 22:52:06 +00:00
|
|
|
return c, nil
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
func (c *Cluster) setupPeerManager() {
|
|
|
|
pm := newPeerManager(c)
|
|
|
|
c.peerManager = pm
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
|
|
|
|
if len(c.config.Peers) > 0 {
|
2017-10-31 10:20:14 +00:00
|
|
|
c.peerManager.addFromMultiaddrs(c.config.Peers, false)
|
2017-02-02 22:52:06 +00:00
|
|
|
} else {
|
2017-10-31 10:20:14 +00:00
|
|
|
c.peerManager.addFromMultiaddrs(c.config.Bootstrap, false)
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Cluster) setupRPC() error {
|
|
|
|
rpcServer := rpc.NewServer(c.host, RPCProtocol)
|
2017-02-08 17:04:08 +00:00
|
|
|
err := rpcServer.RegisterName("Cluster", &RPCAPI{c})
|
2017-02-02 22:52:06 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.rpcServer = rpcServer
|
|
|
|
rpcClient := rpc.NewClientWithServer(c.host, RPCProtocol, rpcServer)
|
|
|
|
c.rpcClient = rpcClient
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
func (c *Cluster) setupConsensus(consensuscfg *raft.Config) error {
|
2017-02-02 22:52:06 +00:00
|
|
|
var startPeers []peer.ID
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
|
|
|
|
if len(c.config.Peers) > 0 {
|
|
|
|
startPeers = peersFromMultiaddrs(c.config.Peers)
|
2017-02-02 22:52:06 +00:00
|
|
|
} else {
|
2017-10-23 11:46:37 +00:00
|
|
|
// start as single cluster before being added
|
|
|
|
// to the bootstrapper peers' cluster.
|
|
|
|
startPeers = []peer.ID{}
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
2017-03-10 16:24:25 +00:00
|
|
|
consensus, err := raft.NewConsensus(
|
2017-02-15 12:40:08 +00:00
|
|
|
append(startPeers, c.id),
|
2017-02-02 22:52:06 +00:00
|
|
|
c.host,
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
consensuscfg,
|
2017-02-02 22:52:06 +00:00
|
|
|
c.state)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Errorf("error creating consensus: %s", err)
|
2017-02-02 22:52:06 +00:00
|
|
|
return err
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.consensus = consensus
|
|
|
|
return nil
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
func (c *Cluster) setupRPCClients() {
|
|
|
|
c.tracker.SetClient(c.rpcClient)
|
|
|
|
c.ipfs.SetClient(c.rpcClient)
|
|
|
|
c.api.SetClient(c.rpcClient)
|
|
|
|
c.consensus.SetClient(c.rpcClient)
|
2017-02-13 15:46:53 +00:00
|
|
|
c.monitor.SetClient(c.rpcClient)
|
|
|
|
c.allocator.SetClient(c.rpcClient)
|
|
|
|
c.informer.SetClient(c.rpcClient)
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2017-04-05 21:29:22 +00:00
|
|
|
// syncWatcher loops and triggers StateSync and SyncAllLocal from time to time
|
|
|
|
func (c *Cluster) syncWatcher() {
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
stateSyncTicker := time.NewTicker(c.config.StateSyncInterval)
|
|
|
|
syncTicker := time.NewTicker(c.config.IPFSSyncInterval)
|
2017-04-05 21:29:22 +00:00
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-stateSyncTicker.C:
|
2017-04-05 21:29:22 +00:00
|
|
|
logger.Debug("auto-triggering StateSync()")
|
2017-02-02 22:52:06 +00:00
|
|
|
c.StateSync()
|
2017-04-05 21:29:22 +00:00
|
|
|
case <-syncTicker.C:
|
|
|
|
logger.Debug("auto-triggering SyncAllLocal()")
|
|
|
|
c.SyncAllLocal()
|
2017-02-02 22:52:06 +00:00
|
|
|
case <-c.ctx.Done():
|
|
|
|
stateSyncTicker.Stop()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-28 15:01:26 +00:00
|
|
|
func (c *Cluster) broadcastMetric(m api.Metric) error {
|
|
|
|
peers := c.peerManager.peers()
|
2017-03-13 14:55:52 +00:00
|
|
|
leader, err := c.consensus.Leader()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2017-07-20 11:18:46 +00:00
|
|
|
// If a peer is down, the rpc call will get locked. Therefore,
|
|
|
|
// we need to do it async. This way we keep broadcasting
|
|
|
|
// even if someone is down. Eventually those requests will
|
|
|
|
// timeout in libp2p and the errors logged.
|
|
|
|
go func() {
|
|
|
|
if leader == c.id {
|
|
|
|
// Leader needs to broadcast its metric to everyone
|
|
|
|
// in case it goes down (new leader will have to detect this node went down)
|
|
|
|
logger.Debugf("Leader %s about to broadcast metric %s to %s. Expires: %s", c.id, m.Name, peers, m.Expire)
|
|
|
|
|
|
|
|
errs := c.multiRPC(peers,
|
|
|
|
"Cluster",
|
|
|
|
"PeerMonitorLogMetric",
|
|
|
|
m,
|
|
|
|
copyEmptyStructToIfaces(make([]struct{}, len(peers), len(peers))))
|
|
|
|
for i, e := range errs {
|
|
|
|
if e != nil {
|
|
|
|
logger.Errorf("error pushing metric to %s: %s", peers[i].Pretty(), e)
|
|
|
|
}
|
2017-03-13 14:55:52 +00:00
|
|
|
}
|
2017-07-20 11:18:46 +00:00
|
|
|
logger.Debugf("Leader %s broadcasted metric %s to %s. Expires: %s", c.id, m.Name, peers, m.Expire)
|
|
|
|
} else {
|
|
|
|
// non-leaders just need to forward their metrics to the leader
|
|
|
|
logger.Debugf("Peer %s about to send metric %s to %s. Expires: %s", c.id, m.Name, leader, m.Expire)
|
|
|
|
|
|
|
|
err := c.rpcClient.Call(leader,
|
|
|
|
"Cluster", "PeerMonitorLogMetric",
|
|
|
|
m, &struct{}{})
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
logger.Debugf("Peer %s sent metric %s to %s. Expires: %s", c.id, m.Name, leader, m.Expire)
|
2017-02-28 15:01:26 +00:00
|
|
|
|
2017-07-20 11:18:46 +00:00
|
|
|
}
|
|
|
|
}()
|
2017-02-28 15:01:26 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
// push metrics loops and pushes metrics to the leader's monitor
|
|
|
|
func (c *Cluster) pushInformerMetrics() {
|
|
|
|
timer := time.NewTimer(0) // fire immediately first
|
2017-10-23 11:46:37 +00:00
|
|
|
// The following control how often to make and log
|
|
|
|
// a retry
|
|
|
|
retries := 0
|
|
|
|
retryDelay := 500 * time.Millisecond
|
|
|
|
retryWarnMod := 60
|
2017-02-13 15:46:53 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
return
|
|
|
|
case <-timer.C:
|
|
|
|
// wait
|
|
|
|
}
|
|
|
|
|
2017-02-28 15:01:26 +00:00
|
|
|
metric := c.informer.GetMetric()
|
|
|
|
metric.Peer = c.id
|
|
|
|
|
|
|
|
err := c.broadcastMetric(metric)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
if err != nil {
|
2017-10-23 11:46:37 +00:00
|
|
|
if (retries % retryWarnMod) == 0 {
|
|
|
|
logger.Errorf("error broadcasting metric: %s", err)
|
|
|
|
retries++
|
|
|
|
}
|
|
|
|
// retry in retryDelay
|
|
|
|
timer.Reset(retryDelay)
|
2017-02-13 15:46:53 +00:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2017-10-23 11:46:37 +00:00
|
|
|
retries = 0
|
|
|
|
// send metric again in TTL/2
|
2017-02-28 15:01:26 +00:00
|
|
|
timer.Reset(metric.GetTTL() / 2)
|
|
|
|
}
|
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-02-28 15:01:26 +00:00
|
|
|
func (c *Cluster) pushPingMetrics() {
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ticker := time.NewTicker(c.config.MonitorPingInterval)
|
2017-02-28 15:01:26 +00:00
|
|
|
for {
|
|
|
|
metric := api.Metric{
|
|
|
|
Name: "ping",
|
|
|
|
Peer: c.id,
|
|
|
|
Valid: true,
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
metric.SetTTLDuration(c.config.MonitorPingInterval * 2)
|
2017-02-28 15:01:26 +00:00
|
|
|
c.broadcastMetric(metric)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-02-28 15:01:26 +00:00
|
|
|
select {
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
return
|
|
|
|
case <-ticker.C:
|
|
|
|
}
|
|
|
|
}
|
2017-07-20 11:18:46 +00:00
|
|
|
logger.Debugf("Peer %s. Finished pushPingMetrics", c.id)
|
2017-02-28 15:01:26 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-02-28 15:01:26 +00:00
|
|
|
// read the alerts channel from the monitor and triggers repins
|
|
|
|
func (c *Cluster) alertsHandler() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
return
|
|
|
|
case alrt := <-c.monitor.Alerts():
|
2017-03-13 14:55:52 +00:00
|
|
|
// only the leader handles alerts
|
|
|
|
leader, err := c.consensus.Leader()
|
|
|
|
if err == nil && leader == c.id {
|
2017-07-20 11:18:46 +00:00
|
|
|
logger.Warningf("Peer %s received alert for %s in %s", c.id, alrt.MetricName, alrt.Peer.Pretty())
|
2017-02-28 15:01:26 +00:00
|
|
|
switch alrt.MetricName {
|
|
|
|
case "ping":
|
|
|
|
c.repinFromPeer(alrt.Peer)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-03 15:45:22 +00:00
|
|
|
// find all Cids pinned to a given peer and triggers re-pins on them.
|
2017-02-28 15:01:26 +00:00
|
|
|
func (c *Cluster) repinFromPeer(p peer.ID) {
|
|
|
|
cState, err := c.consensus.State()
|
|
|
|
if err != nil {
|
|
|
|
logger.Warning(err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
list := cState.List()
|
2017-03-08 17:28:43 +00:00
|
|
|
for _, pin := range list {
|
2017-07-05 14:28:52 +00:00
|
|
|
if containsPeer(pin.Allocations, p) {
|
|
|
|
logger.Infof("repinning %s out of %s", pin.Cid, p.Pretty())
|
|
|
|
c.pin(pin, []peer.ID{p}) // pin blacklisting this peer
|
2017-02-28 15:01:26 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// run provides a cancellable context and launches some goroutines
|
|
|
|
// before signaling readyCh
|
|
|
|
func (c *Cluster) run() {
|
2017-04-05 21:29:22 +00:00
|
|
|
go c.syncWatcher()
|
2017-02-28 15:01:26 +00:00
|
|
|
go c.pushPingMetrics()
|
2017-02-13 15:46:53 +00:00
|
|
|
go c.pushInformerMetrics()
|
2017-02-28 15:01:26 +00:00
|
|
|
go c.alertsHandler()
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
func (c *Cluster) ready() {
|
2017-02-02 22:52:06 +00:00
|
|
|
// We bootstrapped first because with dirty state consensus
|
|
|
|
// may have a peerset and not find a leader so we cannot wait
|
|
|
|
// for it.
|
|
|
|
timer := time.NewTimer(30 * time.Second)
|
|
|
|
select {
|
|
|
|
case <-timer.C:
|
|
|
|
logger.Error("consensus start timed out")
|
|
|
|
c.Shutdown()
|
|
|
|
return
|
|
|
|
case <-c.consensus.Ready():
|
|
|
|
case <-c.ctx.Done():
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Cluster is ready.
|
|
|
|
logger.Info("Cluster Peers (not including ourselves):")
|
|
|
|
peers := c.peerManager.peersAddrs()
|
|
|
|
if len(peers) == 0 {
|
|
|
|
logger.Info(" - No other peers")
|
|
|
|
}
|
|
|
|
for _, a := range c.peerManager.peersAddrs() {
|
|
|
|
logger.Infof(" - %s", a)
|
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
close(c.readyCh)
|
2017-10-30 11:17:39 +00:00
|
|
|
c.readyB = true
|
2017-02-13 15:46:53 +00:00
|
|
|
logger.Info("IPFS Cluster is ready")
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Cluster) bootstrap() bool {
|
|
|
|
// Cases in which we do not bootstrap
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
if len(c.config.Bootstrap) == 0 || len(c.config.Peers) > 0 {
|
2017-02-02 22:52:06 +00:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, b := range c.config.Bootstrap {
|
|
|
|
logger.Infof("Bootstrapping to %s", b)
|
|
|
|
err := c.Join(b)
|
|
|
|
if err == nil {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
return false
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// Ready returns a channel which signals when this peer is
|
|
|
|
// fully initialized (including consensus).
|
|
|
|
func (c *Cluster) Ready() <-chan struct{} {
|
2017-02-02 22:52:06 +00:00
|
|
|
return c.readyCh
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// Shutdown stops the IPFS cluster components
|
|
|
|
func (c *Cluster) Shutdown() error {
|
2016-12-15 13:07:19 +00:00
|
|
|
c.shutdownLock.Lock()
|
|
|
|
defer c.shutdownLock.Unlock()
|
2017-02-02 22:52:06 +00:00
|
|
|
|
2017-10-31 10:20:14 +00:00
|
|
|
if c.shutdownB {
|
|
|
|
logger.Debug("Cluster is already shutdown")
|
2016-12-15 13:07:19 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-10-31 10:20:14 +00:00
|
|
|
logger.Info("shutting down Cluster")
|
2017-02-02 22:52:06 +00:00
|
|
|
|
2017-10-30 11:17:39 +00:00
|
|
|
// Only attempt to leave if consensus is initialized and cluster
|
|
|
|
// was ready at some point. Otherwise, it would mean bootstrap failed.
|
|
|
|
if c.config.LeaveOnShutdown && c.consensus != nil && c.readyB {
|
2017-02-02 22:52:06 +00:00
|
|
|
// best effort
|
2017-10-31 10:20:14 +00:00
|
|
|
logger.Warning("attempting to leave the cluster. This may take some seconds")
|
2017-02-15 12:40:08 +00:00
|
|
|
err := c.consensus.LogRmPeer(c.id)
|
2017-02-02 22:52:06 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("leaving cluster: " + err.Error())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
// Cancel contexts
|
|
|
|
c.cancel()
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
if con := c.consensus; con != nil {
|
|
|
|
if err := con.Shutdown(); err != nil {
|
|
|
|
logger.Errorf("error stopping consensus: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
2017-10-30 12:04:30 +00:00
|
|
|
// Do not save anything if we were not ready
|
|
|
|
if c.readyB {
|
2017-10-31 10:20:14 +00:00
|
|
|
// peers are saved usually on addPeer/rmPeer
|
|
|
|
// c.peerManager.savePeers()
|
2017-10-30 12:04:30 +00:00
|
|
|
c.backupState()
|
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
2017-02-28 15:01:26 +00:00
|
|
|
if err := c.monitor.Shutdown(); err != nil {
|
|
|
|
logger.Errorf("error stopping monitor: %s", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
if err := c.api.Shutdown(); err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error stopping API: %s", err)
|
2016-12-02 18:33:39 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
if err := c.ipfs.Shutdown(); err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error stopping IPFS Connector: %s", err)
|
2016-12-02 18:33:39 +00:00
|
|
|
return err
|
|
|
|
}
|
2016-12-07 16:21:29 +00:00
|
|
|
|
|
|
|
if err := c.tracker.Shutdown(); err != nil {
|
2016-12-15 13:19:41 +00:00
|
|
|
logger.Errorf("error stopping PinTracker: %s", err)
|
2016-12-07 16:21:29 +00:00
|
|
|
return err
|
|
|
|
}
|
2016-12-15 13:07:19 +00:00
|
|
|
c.wg.Wait()
|
2016-12-23 18:35:37 +00:00
|
|
|
c.host.Close() // Shutdown all network services
|
2017-10-31 10:20:14 +00:00
|
|
|
c.shutdownB = true
|
2017-01-30 12:12:25 +00:00
|
|
|
close(c.doneCh)
|
2016-12-09 19:54:46 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// Done provides a way to learn if the Peer has been shutdown
|
|
|
|
// (for example, because it has been removed from the Cluster)
|
|
|
|
func (c *Cluster) Done() <-chan struct{} {
|
|
|
|
return c.doneCh
|
|
|
|
}
|
|
|
|
|
2017-01-25 17:07:19 +00:00
|
|
|
// ID returns information about the Cluster peer
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) ID() api.ID {
|
2017-01-26 18:59:31 +00:00
|
|
|
// ignore error since it is included in response object
|
|
|
|
ipfsID, _ := c.ipfs.ID()
|
|
|
|
var addrs []ma.Multiaddr
|
2017-02-15 12:51:43 +00:00
|
|
|
|
|
|
|
addrsSet := make(map[string]struct{}) // to filter dups
|
2017-01-26 18:59:31 +00:00
|
|
|
for _, addr := range c.host.Addrs() {
|
2017-02-15 12:51:43 +00:00
|
|
|
addrsSet[addr.String()] = struct{}{}
|
|
|
|
}
|
|
|
|
for k := range addrsSet {
|
|
|
|
addr, _ := ma.NewMultiaddr(k)
|
2017-02-15 12:40:08 +00:00
|
|
|
addrs = append(addrs, multiaddrJoin(addr, c.id))
|
2017-01-26 18:59:31 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
return api.ID{
|
2017-02-15 12:40:08 +00:00
|
|
|
ID: c.id,
|
|
|
|
//PublicKey: c.host.Peerstore().PubKey(c.id),
|
2017-01-26 18:59:31 +00:00
|
|
|
Addresses: addrs,
|
2017-02-02 22:52:06 +00:00
|
|
|
ClusterPeers: c.peerManager.peersAddrs(),
|
2017-01-24 15:19:23 +00:00
|
|
|
Version: Version,
|
2017-01-24 15:55:37 +00:00
|
|
|
Commit: Commit,
|
2017-01-24 15:19:23 +00:00
|
|
|
RPCProtocolVersion: RPCProtocol,
|
2017-01-26 18:59:31 +00:00
|
|
|
IPFS: ipfsID,
|
2017-01-24 15:19:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// PeerAdd adds a new peer to this Cluster.
|
|
|
|
//
|
2017-02-02 22:52:06 +00:00
|
|
|
// The new peer must be reachable. It will be added to the
|
|
|
|
// consensus and will receive the shared state (including the
|
|
|
|
// list of peers). The new peer should be a single-peer cluster,
|
|
|
|
// preferable without any relevant state.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) PeerAdd(addr ma.Multiaddr) (api.ID, error) {
|
2017-02-02 22:52:06 +00:00
|
|
|
// starting 10 nodes on the same box for testing
|
|
|
|
// causes deadlock and a global lock here
|
|
|
|
// seems to help.
|
|
|
|
c.paMux.Lock()
|
|
|
|
defer c.paMux.Unlock()
|
|
|
|
logger.Debugf("peerAdd called with %s", addr)
|
|
|
|
pid, decapAddr, err := multiaddrSplit(addr)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{
|
2017-01-30 12:12:25 +00:00
|
|
|
Error: err.Error(),
|
|
|
|
}
|
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Figure out its real address if we have one
|
|
|
|
remoteAddr := getRemoteMultiaddr(c.host, pid, decapAddr)
|
|
|
|
|
2017-10-31 10:20:14 +00:00
|
|
|
err = c.peerManager.addPeer(remoteAddr, false)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{ID: pid, Error: err.Error()}
|
2017-01-30 12:12:25 +00:00
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Figure out our address to that peer. This also
|
|
|
|
// ensures that it is reachable
|
2017-02-08 17:04:08 +00:00
|
|
|
var addrSerial api.MultiaddrSerial
|
2017-02-02 22:52:06 +00:00
|
|
|
err = c.rpcClient.Call(pid, "Cluster",
|
2017-02-15 12:40:08 +00:00
|
|
|
"RemoteMultiaddrForPeer", c.id, &addrSerial)
|
2017-02-02 22:52:06 +00:00
|
|
|
if err != nil {
|
2017-01-30 12:12:25 +00:00
|
|
|
logger.Error(err)
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{ID: pid, Error: err.Error()}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.peerManager.rmPeer(pid, false)
|
2017-01-30 12:12:25 +00:00
|
|
|
return id, err
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Log the new peer in the log so everyone gets it.
|
2017-10-31 10:20:14 +00:00
|
|
|
err = c.consensus.LogAddPeer(remoteAddr) // this will save
|
2017-02-02 22:52:06 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
2017-02-08 17:04:08 +00:00
|
|
|
id := api.ID{ID: pid, Error: err.Error()}
|
2017-02-02 22:52:06 +00:00
|
|
|
c.peerManager.rmPeer(pid, false)
|
|
|
|
return id, err
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
// Send cluster peers to the new peer.
|
|
|
|
clusterPeers := append(c.peerManager.peersAddrs(),
|
|
|
|
addrSerial.ToMultiaddr())
|
|
|
|
err = c.rpcClient.Call(pid,
|
|
|
|
"Cluster",
|
|
|
|
"PeerManagerAddFromMultiaddrs",
|
2017-02-08 17:04:08 +00:00
|
|
|
api.MultiaddrsToSerial(clusterPeers),
|
2017-02-02 22:52:06 +00:00
|
|
|
&struct{}{})
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
2017-02-02 22:52:06 +00:00
|
|
|
logger.Error(err)
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
2017-03-23 18:34:33 +00:00
|
|
|
// Ask the new peer to connect its IPFS daemon to the rest
|
|
|
|
err = c.rpcClient.Call(pid,
|
|
|
|
"Cluster",
|
|
|
|
"IPFSConnectSwarms",
|
|
|
|
struct{}{},
|
|
|
|
&struct{}{})
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
}
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
id, err := c.getIDForPeer(pid)
|
|
|
|
return id, nil
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// PeerRemove removes a peer from this Cluster.
|
|
|
|
//
|
|
|
|
// The peer will be removed from the consensus peer set,
|
2017-02-02 22:52:06 +00:00
|
|
|
// it will be shut down after this happens.
|
|
|
|
func (c *Cluster) PeerRemove(pid peer.ID) error {
|
|
|
|
if !c.peerManager.isPeer(pid) {
|
|
|
|
return fmt.Errorf("%s is not a peer", pid.Pretty())
|
|
|
|
}
|
|
|
|
|
2017-10-31 10:20:14 +00:00
|
|
|
// We need to repin before removing the peer, otherwise, it won't
|
|
|
|
// be able to submit the pins.
|
|
|
|
logger.Infof("re-allocating all CIDs directly associated to %s", pid)
|
2017-07-03 15:45:22 +00:00
|
|
|
c.repinFromPeer(pid)
|
|
|
|
|
2017-02-02 22:52:06 +00:00
|
|
|
err := c.consensus.LogRmPeer(pid)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Join adds this peer to an existing cluster. The calling peer should
|
|
|
|
// be a single-peer cluster node. This is almost equivalent to calling
|
|
|
|
// PeerAdd on the destination cluster.
|
|
|
|
func (c *Cluster) Join(addr ma.Multiaddr) error {
|
|
|
|
logger.Debugf("Join(%s)", addr)
|
|
|
|
|
|
|
|
//if len(c.peerManager.peers()) > 1 {
|
|
|
|
// logger.Error(c.peerManager.peers())
|
|
|
|
// return errors.New("only single-node clusters can be joined")
|
|
|
|
//}
|
|
|
|
|
|
|
|
pid, _, err := multiaddrSplit(addr)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Bootstrap to myself
|
2017-02-15 12:40:08 +00:00
|
|
|
if pid == c.id {
|
2017-02-02 22:52:06 +00:00
|
|
|
return nil
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
// Add peer to peerstore so we can talk to it
|
2017-10-31 10:20:14 +00:00
|
|
|
c.peerManager.addPeer(addr, false)
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
// Note that PeerAdd() on the remote peer will
|
|
|
|
// figure out what our real address is (obviously not
|
2017-10-30 11:17:39 +00:00
|
|
|
// ListenAddr).
|
2017-02-08 17:04:08 +00:00
|
|
|
var myID api.IDSerial
|
2017-02-02 22:52:06 +00:00
|
|
|
err = c.rpcClient.Call(pid,
|
|
|
|
"Cluster",
|
|
|
|
"PeerAdd",
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
api.MultiaddrToSerial(
|
|
|
|
multiaddrJoin(c.config.ListenAddr, c.id)),
|
2017-02-02 22:52:06 +00:00
|
|
|
&myID)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
|
|
|
|
// wait for leader and for state to catch up
|
|
|
|
// then sync
|
|
|
|
err = c.consensus.WaitForSync()
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
c.StateSync()
|
|
|
|
|
|
|
|
logger.Infof("joined %s's cluster", addr)
|
2017-01-30 12:12:25 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-20 18:51:13 +00:00
|
|
|
// StateSync syncs the consensus state to the Pin Tracker, ensuring
|
|
|
|
// that every Cid that should be tracked is tracked. It returns
|
|
|
|
// PinInfo for Cids which were added or deleted.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) StateSync() ([]api.PinInfo, error) {
|
2016-12-09 19:54:46 +00:00
|
|
|
cState, err := c.consensus.State()
|
|
|
|
if err != nil {
|
2016-12-15 18:08:46 +00:00
|
|
|
return nil, err
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-01-23 22:58:04 +00:00
|
|
|
logger.Debug("syncing state to tracker")
|
2017-02-13 15:46:53 +00:00
|
|
|
clusterPins := cState.List()
|
2016-12-20 18:51:13 +00:00
|
|
|
var changed []*cid.Cid
|
|
|
|
|
|
|
|
// Track items which are not tracked
|
2017-03-08 15:57:27 +00:00
|
|
|
for _, pin := range clusterPins {
|
|
|
|
if c.tracker.Status(pin.Cid).Status == api.TrackerStatusUnpinned {
|
|
|
|
changed = append(changed, pin.Cid)
|
|
|
|
go c.tracker.Track(pin)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Untrack items which should not be tracked
|
2017-01-25 18:38:23 +00:00
|
|
|
for _, p := range c.tracker.StatusAll() {
|
2017-02-13 15:46:53 +00:00
|
|
|
if !cState.Has(p.Cid) {
|
2017-02-08 17:04:08 +00:00
|
|
|
changed = append(changed, p.Cid)
|
|
|
|
go c.tracker.Untrack(p.Cid)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
var infos []api.PinInfo
|
2016-12-19 17:35:24 +00:00
|
|
|
for _, h := range changed {
|
2017-01-25 18:38:23 +00:00
|
|
|
infos = append(infos, c.tracker.Status(h))
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
return infos, nil
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// StatusAll returns the GlobalPinInfo for all tracked Cids. If an error
|
|
|
|
// happens, the slice will contain as much information as could be fetched.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) StatusAll() ([]api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoSlice("TrackerStatusAll")
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// Status returns the GlobalPinInfo for a given Cid. If an error happens,
|
2016-12-23 18:35:37 +00:00
|
|
|
// the GlobalPinInfo should contain as much information as could be fetched.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Status(h *cid.Cid) (api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoCid("TrackerStatus", h)
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncAllLocal makes sure that the current state for all tracked items
|
|
|
|
// matches the state reported by the IPFS daemon.
|
2016-12-20 18:51:13 +00:00
|
|
|
//
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncAllLocal returns the list of PinInfo that where updated because of
|
|
|
|
// the operation, along with those in error states.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) SyncAllLocal() ([]api.PinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
syncedItems, err := c.tracker.SyncAll()
|
|
|
|
// Despite errors, tracker provides synced items that we can provide.
|
|
|
|
// They encapsulate the error.
|
2017-01-25 17:07:19 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error("tracker.Sync() returned with error: ", err)
|
|
|
|
logger.Error("Is the ipfs daemon running?")
|
2016-12-09 19:54:46 +00:00
|
|
|
}
|
2017-01-25 18:38:23 +00:00
|
|
|
return syncedItems, err
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// SyncLocal performs a local sync operation for the given Cid. This will
|
|
|
|
// tell the tracker to verify the status of the Cid against the IPFS daemon.
|
|
|
|
// It returns the updated PinInfo for the Cid.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) SyncLocal(h *cid.Cid) (api.PinInfo, error) {
|
2016-12-20 18:51:13 +00:00
|
|
|
var err error
|
2017-01-25 18:38:23 +00:00
|
|
|
pInfo, err := c.tracker.Sync(h)
|
2017-01-25 17:07:19 +00:00
|
|
|
// Despite errors, trackers provides an updated PinInfo so
|
|
|
|
// we just log it.
|
|
|
|
if err != nil {
|
|
|
|
logger.Error("tracker.SyncCid() returned with error: ", err)
|
|
|
|
logger.Error("Is the ipfs daemon running?")
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
2017-01-25 18:38:23 +00:00
|
|
|
return pInfo, err
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// SyncAll triggers LocalSync() operations in all cluster peers.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) SyncAll() ([]api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoSlice("SyncAllLocal")
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
// Sync triggers a LocalSyncCid() operation for a given Cid
|
2017-01-26 18:59:31 +00:00
|
|
|
// in all cluster peers.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Sync(h *cid.Cid) (api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoCid("SyncLocal", h)
|
|
|
|
}
|
|
|
|
|
|
|
|
// RecoverLocal triggers a recover operation for a given Cid
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) RecoverLocal(h *cid.Cid) (api.PinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.tracker.Recover(h)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Recover triggers a recover operation for a given Cid in all
|
2017-01-26 18:59:31 +00:00
|
|
|
// cluster peers.
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Recover(h *cid.Cid) (api.GlobalPinInfo, error) {
|
2017-01-25 18:38:23 +00:00
|
|
|
return c.globalPinInfoCid("TrackerRecover", h)
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Pins returns the list of Cids managed by Cluster and which are part
|
|
|
|
// of the current global state. This is the source of truth as to which
|
2017-04-06 02:27:02 +00:00
|
|
|
// pins are managed and their allocation, but does not indicate if
|
|
|
|
// the item is successfully pinned. For that, use StatusAll().
|
2017-03-08 15:57:27 +00:00
|
|
|
func (c *Cluster) Pins() []api.Pin {
|
2016-12-15 18:08:46 +00:00
|
|
|
cState, err := c.consensus.State()
|
|
|
|
if err != nil {
|
2017-02-01 17:16:09 +00:00
|
|
|
logger.Error(err)
|
2017-03-08 15:57:27 +00:00
|
|
|
return []api.Pin{}
|
2016-12-15 18:08:46 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
return cState.List()
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
|
2017-04-06 02:27:02 +00:00
|
|
|
// PinGet returns information for a single Cid managed by Cluster.
|
|
|
|
// The information is obtained from the current global state. The
|
|
|
|
// returned api.Pin provides information about the allocations
|
|
|
|
// assigned for the requested Cid, but does not provide indicate if
|
|
|
|
// the item is successfully pinned. For that, use Status(). PinGet
|
|
|
|
// returns an error if the given Cid is not part of the global state.
|
|
|
|
func (c *Cluster) PinGet(h *cid.Cid) (api.Pin, error) {
|
|
|
|
cState, err := c.consensus.State()
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return api.Pin{}, err
|
|
|
|
}
|
|
|
|
pin := cState.Get(h)
|
|
|
|
if pin.Cid == nil {
|
|
|
|
return pin, errors.New("Cid is not part of the global state")
|
|
|
|
}
|
|
|
|
return pin, nil
|
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// Pin makes the cluster Pin a Cid. This implies adding the Cid
|
|
|
|
// to the IPFS Cluster peers shared-state. Depending on the cluster
|
2016-12-16 11:40:28 +00:00
|
|
|
// pinning strategy, the PinTracker may then request the IPFS daemon
|
2017-01-23 13:01:49 +00:00
|
|
|
// to pin the Cid.
|
2016-12-02 18:33:39 +00:00
|
|
|
//
|
|
|
|
// Pin returns an error if the operation could not be persisted
|
|
|
|
// to the global state. Pin does not reflect the success or failure
|
|
|
|
// of underlying IPFS daemon pinning operations.
|
2017-03-08 17:28:43 +00:00
|
|
|
func (c *Cluster) Pin(pin api.Pin) error {
|
2017-07-03 15:45:22 +00:00
|
|
|
return c.pin(pin, []peer.ID{})
|
|
|
|
}
|
|
|
|
|
|
|
|
// pin performs the actual pinning and supports a blacklist to be
|
|
|
|
// able to evacuate a node.
|
|
|
|
func (c *Cluster) pin(pin api.Pin, blacklist []peer.ID) error {
|
2017-03-08 17:28:43 +00:00
|
|
|
rpl := pin.ReplicationFactor
|
|
|
|
if rpl == 0 {
|
|
|
|
rpl = c.config.ReplicationFactor
|
|
|
|
pin.ReplicationFactor = rpl
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
switch {
|
|
|
|
case rpl == 0:
|
|
|
|
return errors.New("replication factor is 0")
|
|
|
|
case rpl < 0:
|
2017-03-29 20:52:13 +00:00
|
|
|
pin.Allocations = []peer.ID{}
|
2017-03-08 17:28:43 +00:00
|
|
|
logger.Infof("IPFS cluster pinning %s everywhere:", pin.Cid)
|
2017-02-13 15:46:53 +00:00
|
|
|
case rpl > 0:
|
2017-07-03 15:45:22 +00:00
|
|
|
allocs, err := c.allocate(pin.Cid, pin.ReplicationFactor, blacklist)
|
2017-02-13 15:46:53 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2017-03-08 15:57:27 +00:00
|
|
|
pin.Allocations = allocs
|
2017-03-08 17:28:43 +00:00
|
|
|
logger.Infof("IPFS cluster pinning %s on %s:", pin.Cid, pin.Allocations)
|
2017-02-28 15:01:26 +00:00
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
2017-03-08 15:57:27 +00:00
|
|
|
err := c.consensus.LogPin(pin)
|
2016-12-23 18:35:37 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unpin makes the cluster Unpin a Cid. This implies adding the Cid
|
2017-01-23 13:01:49 +00:00
|
|
|
// to the IPFS Cluster peers shared-state.
|
2016-12-02 18:33:39 +00:00
|
|
|
//
|
|
|
|
// Unpin returns an error if the operation could not be persisted
|
|
|
|
// to the global state. Unpin does not reflect the success or failure
|
|
|
|
// of underlying IPFS daemon unpinning operations.
|
|
|
|
func (c *Cluster) Unpin(h *cid.Cid) error {
|
2017-02-28 15:01:26 +00:00
|
|
|
logger.Info("IPFS cluster unpinning:", h)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-03-08 15:57:27 +00:00
|
|
|
pin := api.Pin{
|
2017-02-13 15:46:53 +00:00
|
|
|
Cid: h,
|
|
|
|
}
|
|
|
|
|
2017-03-08 15:57:27 +00:00
|
|
|
err := c.consensus.LogUnpin(pin)
|
2016-12-23 18:35:37 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
2016-12-02 18:33:39 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Version returns the current IPFS Cluster version
|
|
|
|
func (c *Cluster) Version() string {
|
|
|
|
return Version
|
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
// Peers returns the IDs of the members of this Cluster
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) Peers() []api.ID {
|
2017-01-30 12:12:25 +00:00
|
|
|
members := c.peerManager.peers()
|
2017-02-08 17:04:08 +00:00
|
|
|
peersSerial := make([]api.IDSerial, len(members), len(members))
|
|
|
|
peers := make([]api.ID, len(members), len(members))
|
2017-01-26 18:59:31 +00:00
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
errs := c.multiRPC(members, "Cluster", "ID", struct{}{},
|
|
|
|
copyIDSerialsToIfaces(peersSerial))
|
2017-01-26 18:59:31 +00:00
|
|
|
|
|
|
|
for i, err := range errs {
|
|
|
|
if err != nil {
|
|
|
|
peersSerial[i].ID = peer.IDB58Encode(members[i])
|
|
|
|
peersSerial[i].Error = err.Error()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for i, ps := range peersSerial {
|
|
|
|
peers[i] = ps.ToID()
|
|
|
|
}
|
|
|
|
return peers
|
|
|
|
}
|
|
|
|
|
2016-12-02 18:33:39 +00:00
|
|
|
// makeHost makes a libp2p-host
|
2016-12-15 18:08:46 +00:00
|
|
|
func makeHost(ctx context.Context, cfg *Config) (host.Host, error) {
|
2016-12-02 18:33:39 +00:00
|
|
|
ps := peerstore.NewPeerstore()
|
2017-01-23 17:38:59 +00:00
|
|
|
privateKey := cfg.PrivateKey
|
2016-12-02 18:33:39 +00:00
|
|
|
publicKey := privateKey.GetPublic()
|
|
|
|
|
2017-06-22 17:21:05 +00:00
|
|
|
var protec ipnet.Protector
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
if len(cfg.Secret) != 0 {
|
2017-06-30 00:40:06 +00:00
|
|
|
var err error
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
clusterKey, err := clusterSecretToKey(cfg.Secret)
|
2017-07-03 20:00:01 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2017-07-04 20:39:10 +00:00
|
|
|
protec, err = pnet.NewProtector(strings.NewReader(clusterKey))
|
2017-06-22 17:21:05 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// this is in go-ipfs, not sure whether we want something like it here
|
|
|
|
/* go func() {
|
|
|
|
t := time.NewTicker(30 * time.Second)
|
|
|
|
<-t.C // swallow one tick
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-t.C:
|
|
|
|
if ph := cfg.Host; ph != nil {
|
|
|
|
if len(ph.Network().Peers()) == 0 {
|
|
|
|
log.Warning("We are in a private network and have no peers.")
|
|
|
|
log.Warning("This might be a configuration mistake.")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
case <-n.Process().Closing:
|
|
|
|
t.Stop()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()*/
|
|
|
|
}
|
|
|
|
|
2017-01-23 17:38:59 +00:00
|
|
|
if err := ps.AddPubKey(cfg.ID, publicKey); err != nil {
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-01-23 17:38:59 +00:00
|
|
|
if err := ps.AddPrivKey(cfg.ID, privateKey); err != nil {
|
2016-12-02 18:33:39 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-06-22 17:21:05 +00:00
|
|
|
network, err := swarm.NewNetworkWithProtector(
|
2016-12-02 18:33:39 +00:00
|
|
|
ctx,
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
[]ma.Multiaddr{cfg.ListenAddr},
|
2017-01-23 17:38:59 +00:00
|
|
|
cfg.ID,
|
2016-12-02 18:33:39 +00:00
|
|
|
ps,
|
2017-06-22 17:21:05 +00:00
|
|
|
protec,
|
2016-12-02 18:33:39 +00:00
|
|
|
nil,
|
|
|
|
)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
bhost := basichost.New(network)
|
|
|
|
return bhost, nil
|
|
|
|
}
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
// Perform an RPC request to multiple destinations
|
2016-12-23 18:35:37 +00:00
|
|
|
func (c *Cluster) multiRPC(dests []peer.ID, svcName, svcMethod string, args interface{}, reply []interface{}) []error {
|
|
|
|
if len(dests) != len(reply) {
|
2017-01-23 13:21:26 +00:00
|
|
|
panic("must have matching dests and replies")
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
errs := make([]error, len(dests), len(dests))
|
|
|
|
|
2016-12-28 15:25:24 +00:00
|
|
|
for i := range dests {
|
2016-12-23 18:35:37 +00:00
|
|
|
wg.Add(1)
|
|
|
|
go func(i int) {
|
|
|
|
defer wg.Done()
|
|
|
|
err := c.rpcClient.Call(
|
|
|
|
dests[i],
|
|
|
|
svcName,
|
|
|
|
svcMethod,
|
|
|
|
args,
|
|
|
|
reply[i])
|
|
|
|
errs[i] = err
|
|
|
|
}(i)
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
return errs
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2016-12-23 18:35:37 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) globalPinInfoCid(method string, h *cid.Cid) (api.GlobalPinInfo, error) {
|
|
|
|
pin := api.GlobalPinInfo{
|
2017-01-25 17:07:19 +00:00
|
|
|
Cid: h,
|
2017-02-08 17:04:08 +00:00
|
|
|
PeerMap: make(map[peer.ID]api.PinInfo),
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
members := c.peerManager.peers()
|
2017-02-08 17:04:08 +00:00
|
|
|
replies := make([]api.PinInfoSerial, len(members), len(members))
|
2017-03-08 15:57:27 +00:00
|
|
|
arg := api.Pin{
|
2017-02-08 17:04:08 +00:00
|
|
|
Cid: h,
|
|
|
|
}
|
|
|
|
errs := c.multiRPC(members,
|
|
|
|
"Cluster",
|
|
|
|
method, arg.ToSerial(),
|
|
|
|
copyPinInfoSerialToIfaces(replies))
|
|
|
|
|
|
|
|
for i, rserial := range replies {
|
2017-06-21 12:16:28 +00:00
|
|
|
e := errs[i]
|
|
|
|
|
|
|
|
// Potentially rserial is empty. But ToPinInfo ignores all
|
|
|
|
// errors from underlying libraries. In that case .Status
|
|
|
|
// will be TrackerStatusBug (0)
|
2017-02-08 17:04:08 +00:00
|
|
|
r := rserial.ToPinInfo()
|
2017-06-21 12:16:28 +00:00
|
|
|
|
|
|
|
// No error. Parse and continue
|
|
|
|
if e == nil {
|
|
|
|
pin.PeerMap[members[i]] = r
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// Deal with error cases (err != nil): wrap errors in PinInfo
|
|
|
|
|
|
|
|
// In this case, we had no answer at all. The contacted peer
|
|
|
|
// must be offline or unreachable.
|
|
|
|
if r.Status == api.TrackerStatusBug {
|
|
|
|
logger.Errorf("%s: error in broadcast response from %s: %s ", c.id, members[i], e)
|
|
|
|
pin.PeerMap[members[i]] = api.PinInfo{
|
|
|
|
Cid: h,
|
|
|
|
Peer: members[i],
|
|
|
|
Status: api.TrackerStatusClusterError,
|
|
|
|
TS: time.Now(),
|
|
|
|
Error: e.Error(),
|
2017-01-23 23:52:42 +00:00
|
|
|
}
|
2017-06-21 12:16:28 +00:00
|
|
|
} else { // there was an rpc error, but got a valid response :S
|
|
|
|
r.Error = e.Error()
|
|
|
|
pin.PeerMap[members[i]] = r
|
|
|
|
// unlikely to come down this path
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
2016-12-28 15:25:24 +00:00
|
|
|
|
2017-01-23 23:52:42 +00:00
|
|
|
return pin, nil
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) globalPinInfoSlice(method string) ([]api.GlobalPinInfo, error) {
|
|
|
|
var infos []api.GlobalPinInfo
|
|
|
|
fullMap := make(map[string]api.GlobalPinInfo)
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-01-30 12:12:25 +00:00
|
|
|
members := c.peerManager.peers()
|
2017-02-08 17:04:08 +00:00
|
|
|
replies := make([][]api.PinInfoSerial, len(members), len(members))
|
|
|
|
errs := c.multiRPC(members,
|
|
|
|
"Cluster",
|
|
|
|
method, struct{}{},
|
|
|
|
copyPinInfoSerialSliceToIfaces(replies))
|
2016-12-20 18:51:13 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
mergePins := func(pins []api.PinInfoSerial) {
|
|
|
|
for _, pserial := range pins {
|
|
|
|
p := pserial.ToPinInfo()
|
|
|
|
item, ok := fullMap[pserial.Cid]
|
2016-12-20 18:51:13 +00:00
|
|
|
if !ok {
|
2017-02-08 17:04:08 +00:00
|
|
|
fullMap[pserial.Cid] = api.GlobalPinInfo{
|
|
|
|
Cid: p.Cid,
|
|
|
|
PeerMap: map[peer.ID]api.PinInfo{
|
2016-12-20 18:51:13 +00:00
|
|
|
p.Peer: p,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
} else {
|
2017-01-25 17:07:19 +00:00
|
|
|
item.PeerMap[p.Peer] = p
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-24 00:09:27 +00:00
|
|
|
erroredPeers := make(map[peer.ID]string)
|
2016-12-23 18:35:37 +00:00
|
|
|
for i, r := range replies {
|
2017-01-25 17:07:19 +00:00
|
|
|
if e := errs[i]; e != nil { // This error must come from not being able to contact that cluster member
|
2017-02-15 12:40:08 +00:00
|
|
|
logger.Errorf("%s: error in broadcast response from %s: %s ", c.id, members[i], e)
|
2017-01-24 00:09:27 +00:00
|
|
|
erroredPeers[members[i]] = e.Error()
|
2017-01-23 23:52:42 +00:00
|
|
|
} else {
|
|
|
|
mergePins(r)
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-24 00:09:27 +00:00
|
|
|
// Merge any errors
|
|
|
|
for p, msg := range erroredPeers {
|
2017-02-08 17:04:08 +00:00
|
|
|
for cidStr := range fullMap {
|
|
|
|
c, _ := cid.Decode(cidStr)
|
|
|
|
fullMap[cidStr].PeerMap[p] = api.PinInfo{
|
|
|
|
Cid: c,
|
2017-01-24 00:09:27 +00:00
|
|
|
Peer: p,
|
2017-02-08 17:04:08 +00:00
|
|
|
Status: api.TrackerStatusClusterError,
|
2017-01-24 00:09:27 +00:00
|
|
|
TS: time.Now(),
|
|
|
|
Error: msg,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-20 18:51:13 +00:00
|
|
|
for _, v := range fullMap {
|
|
|
|
infos = append(infos, v)
|
|
|
|
}
|
|
|
|
|
2017-01-23 23:52:42 +00:00
|
|
|
return infos, nil
|
2016-12-20 18:51:13 +00:00
|
|
|
}
|
2017-01-23 19:29:05 +00:00
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
func (c *Cluster) getIDForPeer(pid peer.ID) (api.ID, error) {
|
|
|
|
idSerial := api.ID{ID: pid}.ToSerial()
|
2017-02-02 22:52:06 +00:00
|
|
|
err := c.rpcClient.Call(
|
|
|
|
pid, "Cluster", "ID", struct{}{}, &idSerial)
|
|
|
|
id := idSerial.ToID()
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
id.Error = err.Error()
|
2017-01-23 19:29:05 +00:00
|
|
|
}
|
2017-02-02 22:52:06 +00:00
|
|
|
return id, err
|
2017-01-23 19:29:05 +00:00
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
// allocate finds peers to allocate a hash using the informer and the monitor
|
|
|
|
// it should only be used with a positive replication factor
|
2017-07-03 15:45:22 +00:00
|
|
|
func (c *Cluster) allocate(hash *cid.Cid, repl int, blacklist []peer.ID) ([]peer.ID, error) {
|
2017-03-08 17:28:43 +00:00
|
|
|
if repl <= 0 {
|
2017-02-13 15:46:53 +00:00
|
|
|
return nil, errors.New("cannot decide allocation for replication factor <= 0")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Figure out who is currently holding this
|
2017-07-03 15:45:22 +00:00
|
|
|
var pinAllocations []peer.ID
|
2017-02-13 15:46:53 +00:00
|
|
|
st, err := c.consensus.State()
|
|
|
|
if err != nil {
|
|
|
|
// no state we assume it is empty. If there was other
|
|
|
|
// problem, we would fail to commit anyway.
|
2017-07-03 15:45:22 +00:00
|
|
|
pinAllocations = []peer.ID{}
|
2017-02-13 15:46:53 +00:00
|
|
|
} else {
|
2017-03-08 15:57:27 +00:00
|
|
|
pin := st.Get(hash)
|
2017-07-03 15:45:22 +00:00
|
|
|
pinAllocations = pin.Allocations
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
2017-07-03 15:45:22 +00:00
|
|
|
// Get the LastMetrics from the leading monitor. They are the last
|
|
|
|
// valid metrics from current cluster peers
|
|
|
|
var metrics []api.Metric
|
2017-02-13 15:46:53 +00:00
|
|
|
metricName := c.informer.Name()
|
|
|
|
l, err := c.consensus.Leader()
|
|
|
|
if err != nil {
|
|
|
|
return nil, errors.New("cannot determine leading Monitor")
|
|
|
|
}
|
2017-07-03 15:45:22 +00:00
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
err = c.rpcClient.Call(l,
|
|
|
|
"Cluster", "PeerMonitorLastMetrics",
|
|
|
|
metricName,
|
|
|
|
&metrics)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2017-07-03 15:45:22 +00:00
|
|
|
// We must divide the metrics between current and candidates
|
|
|
|
current := make(map[peer.ID]api.Metric)
|
|
|
|
candidates := make(map[peer.ID]api.Metric)
|
|
|
|
validAllocations := make([]peer.ID, 0, len(pinAllocations))
|
2017-02-13 15:46:53 +00:00
|
|
|
for _, m := range metrics {
|
2017-07-03 15:45:22 +00:00
|
|
|
if m.Discard() || containsPeer(blacklist, m.Peer) {
|
|
|
|
// blacklisted peers do not exist for us
|
2017-02-13 15:46:53 +00:00
|
|
|
continue
|
2017-07-03 15:45:22 +00:00
|
|
|
} else if containsPeer(pinAllocations, m.Peer) {
|
|
|
|
current[m.Peer] = m
|
|
|
|
validAllocations = append(validAllocations, m.Peer)
|
|
|
|
} else {
|
|
|
|
candidates[m.Peer] = m
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-03 15:45:22 +00:00
|
|
|
currentValid := len(validAllocations)
|
|
|
|
candidatesValid := len(candidates)
|
|
|
|
needed := repl - currentValid
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-07-03 15:45:22 +00:00
|
|
|
logger.Debugf("allocate: Valid allocations: %d", currentValid)
|
|
|
|
logger.Debugf("allocate: Valid candidates: %d", candidatesValid)
|
2017-03-09 13:44:14 +00:00
|
|
|
logger.Debugf("allocate: Needed: %d", needed)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-03-09 13:44:14 +00:00
|
|
|
// If needed == 0, we don't need anything. If needed < 0, we are
|
|
|
|
// reducing the replication factor
|
|
|
|
switch {
|
2017-07-03 15:45:22 +00:00
|
|
|
|
|
|
|
case needed <= 0: // set the allocations to the needed ones
|
2017-03-09 13:44:14 +00:00
|
|
|
return validAllocations[0 : len(validAllocations)+needed], nil
|
2017-07-05 14:28:52 +00:00
|
|
|
case candidatesValid < needed:
|
|
|
|
err = logError(
|
2017-07-05 17:08:37 +00:00
|
|
|
"not enough candidates to allocate %s. Needed: %d. Got: %d (%s)",
|
|
|
|
hash, needed, candidatesValid, candidates)
|
2017-07-05 14:28:52 +00:00
|
|
|
return nil, err
|
2017-03-09 13:44:14 +00:00
|
|
|
default:
|
2017-07-03 15:45:22 +00:00
|
|
|
// this will return candidate peers in order of
|
|
|
|
// preference according to the allocator.
|
|
|
|
candidateAllocs, err := c.allocator.Allocate(hash, current, candidates)
|
2017-03-09 13:44:14 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, logError(err.Error())
|
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-03-09 13:44:14 +00:00
|
|
|
logger.Debugf("allocate: candidate allocations: %s", candidateAllocs)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2017-03-09 13:44:14 +00:00
|
|
|
// we don't have enough peers to pin
|
2017-07-05 17:08:37 +00:00
|
|
|
if got := len(candidateAllocs); got < needed {
|
2017-07-03 15:45:22 +00:00
|
|
|
err = logError(
|
2017-07-05 17:08:37 +00:00
|
|
|
"cannot find enough allocations for %s. Needed: %d. Got: %d (%s)",
|
|
|
|
hash, needed, got, candidateAllocs)
|
2017-03-09 13:44:14 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// the new allocations = the valid ones we had + the needed ones
|
|
|
|
return append(validAllocations, candidateAllocs[0:needed]...), nil
|
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
2017-02-27 16:28:41 +00:00
|
|
|
|
|
|
|
func (c *Cluster) backupState() {
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
if c.config.BaseDir == "" {
|
|
|
|
logger.Warning("ClusterConfig BaseDir unset. Skipping backup")
|
2017-02-27 16:28:41 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2017-10-31 10:20:14 +00:00
|
|
|
folder := filepath.Join(c.config.BaseDir, "backups")
|
|
|
|
err := os.MkdirAll(folder, 0700)
|
2017-02-27 16:28:41 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
logger.Error("skipping backup")
|
|
|
|
return
|
|
|
|
}
|
|
|
|
fname := time.Now().UTC().Format("20060102_15:04:05")
|
2017-10-31 10:20:14 +00:00
|
|
|
f, err := os.Create(filepath.Join(folder, fname))
|
2017-02-27 16:28:41 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return
|
|
|
|
}
|
2017-10-30 13:16:44 +00:00
|
|
|
defer f.Close()
|
|
|
|
|
2017-02-27 16:28:41 +00:00
|
|
|
err = c.state.Snapshot(f)
|
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|