2016-12-08 16:24:38 +00:00
|
|
|
package ipfscluster
|
|
|
|
|
|
|
|
import (
|
2018-03-29 18:18:15 +00:00
|
|
|
"context"
|
2018-06-07 02:19:15 +00:00
|
|
|
"errors"
|
2017-11-29 16:49:03 +00:00
|
|
|
"flag"
|
2016-12-21 13:30:54 +00:00
|
|
|
"fmt"
|
|
|
|
"math/rand"
|
|
|
|
"os"
|
2017-03-09 13:44:14 +00:00
|
|
|
"sort"
|
2017-02-13 15:46:53 +00:00
|
|
|
"strings"
|
2016-12-21 13:30:54 +00:00
|
|
|
"sync"
|
2016-12-08 16:24:38 +00:00
|
|
|
"testing"
|
|
|
|
"time"
|
2016-12-09 19:54:46 +00:00
|
|
|
|
2017-09-01 12:09:37 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/allocator/descendalloc"
|
2017-02-08 17:04:08 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api/rest"
|
|
|
|
"github.com/ipfs/ipfs-cluster/consensus/raft"
|
2017-03-27 13:07:12 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/informer/disk"
|
2017-03-14 15:10:45 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/ipfsconn/ipfshttp"
|
2018-05-08 12:39:45 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/monitor/basic"
|
2018-05-07 12:34:11 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/monitor/pubsubmon"
|
2018-06-27 04:03:15 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/observations"
|
2017-03-14 15:10:45 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/pintracker/maptracker"
|
2018-06-07 02:19:15 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/pintracker/stateless"
|
2017-03-10 16:24:25 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/state"
|
2017-02-09 15:29:17 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/state/mapstate"
|
|
|
|
"github.com/ipfs/ipfs-cluster/test"
|
2018-12-18 14:44:11 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/version"
|
2017-02-08 17:04:08 +00:00
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
cid "github.com/ipfs/go-cid"
|
|
|
|
crypto "github.com/libp2p/go-libp2p-crypto"
|
2018-03-29 18:18:15 +00:00
|
|
|
host "github.com/libp2p/go-libp2p-host"
|
2016-12-16 11:40:28 +00:00
|
|
|
peer "github.com/libp2p/go-libp2p-peer"
|
2018-04-05 14:09:41 +00:00
|
|
|
peerstore "github.com/libp2p/go-libp2p-peerstore"
|
2017-01-23 17:38:59 +00:00
|
|
|
ma "github.com/multiformats/go-multiaddr"
|
2016-12-08 16:24:38 +00:00
|
|
|
)
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
var (
|
|
|
|
// number of clusters to create
|
2018-03-29 18:18:15 +00:00
|
|
|
nClusters = 5
|
2016-12-21 13:30:54 +00:00
|
|
|
|
|
|
|
// number of pins to pin/unpin/check
|
2018-03-16 16:37:39 +00:00
|
|
|
nPins = 100
|
2018-01-15 09:49:07 +00:00
|
|
|
|
2018-06-07 02:19:15 +00:00
|
|
|
logLevel = "CRITICAL"
|
|
|
|
customLogLvlFacilities = logFacilities{}
|
2016-12-21 13:30:54 +00:00
|
|
|
|
2018-05-08 12:39:45 +00:00
|
|
|
pmonitor = "pubsub"
|
2018-08-14 17:18:26 +00:00
|
|
|
ptracker = "map"
|
2018-05-08 12:39:45 +00:00
|
|
|
|
2018-01-25 22:34:06 +00:00
|
|
|
// When testing with fixed ports...
|
|
|
|
// clusterPort = 10000
|
|
|
|
// apiPort = 10100
|
|
|
|
// ipfsProxyPort = 10200
|
2016-12-21 13:30:54 +00:00
|
|
|
)
|
|
|
|
|
2018-06-07 02:19:15 +00:00
|
|
|
type logFacilities []string
|
|
|
|
|
|
|
|
// String is the method to format the flag's value, part of the flag.Value interface.
|
|
|
|
func (lg *logFacilities) String() string {
|
|
|
|
return fmt.Sprint(*lg)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set is the method to set the flag value, part of the flag.Value interface.
|
|
|
|
func (lg *logFacilities) Set(value string) error {
|
|
|
|
if len(*lg) > 0 {
|
|
|
|
return errors.New("logFacilities flag already set")
|
|
|
|
}
|
|
|
|
for _, lf := range strings.Split(value, ",") {
|
|
|
|
*lg = append(*lg, lf)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
func init() {
|
2018-06-07 02:19:15 +00:00
|
|
|
flag.Var(&customLogLvlFacilities, "logfacs", "use -logLevel for only the following log facilities; comma-separated")
|
2018-01-15 09:49:07 +00:00
|
|
|
flag.StringVar(&logLevel, "loglevel", logLevel, "default log level for tests")
|
|
|
|
flag.IntVar(&nClusters, "nclusters", nClusters, "number of clusters to use")
|
|
|
|
flag.IntVar(&nPins, "npins", nPins, "number of pins to pin/unpin/check")
|
2018-05-08 12:39:45 +00:00
|
|
|
flag.StringVar(&pmonitor, "monitor", pmonitor, "monitor implementation")
|
2018-06-07 02:19:15 +00:00
|
|
|
flag.StringVar(&ptracker, "tracker", ptracker, "tracker implementation")
|
2018-01-15 09:49:07 +00:00
|
|
|
flag.Parse()
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
rand.Seed(time.Now().UnixNano())
|
2018-01-15 09:49:07 +00:00
|
|
|
|
2018-06-11 01:57:42 +00:00
|
|
|
if len(customLogLvlFacilities) <= 0 {
|
2018-06-07 02:19:15 +00:00
|
|
|
for f := range LoggingFacilities {
|
|
|
|
SetFacilityLogLevel(f, logLevel)
|
|
|
|
}
|
|
|
|
|
|
|
|
for f := range LoggingFacilitiesExtra {
|
|
|
|
SetFacilityLogLevel(f, logLevel)
|
|
|
|
}
|
2018-01-15 09:49:07 +00:00
|
|
|
}
|
|
|
|
|
2018-06-07 02:19:15 +00:00
|
|
|
for _, f := range customLogLvlFacilities {
|
|
|
|
if _, ok := LoggingFacilities[f]; ok {
|
|
|
|
SetFacilityLogLevel(f, logLevel)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if _, ok := LoggingFacilitiesExtra[f]; ok {
|
|
|
|
SetFacilityLogLevel(f, logLevel)
|
|
|
|
continue
|
|
|
|
}
|
Fix: maptracker race issues
This commit attempts to fix race issues in the maptracker since the
introduction of the OperationTracker.
There were two main problems:
* Duplicity tracking the state both in the state map and the opTracker
* Non atomiciy of operations with different threads being able to affect
other threads operations.
A test performing random Track/Untracks on the same Cid quickly showed
that items would sometimes stay as pin_queued or pin_unqueued. That happened
because operations could be cancelled under the hood by a different request,
while leaving the map status untouched.
It was not simply to deal with this issues without a refactoring.
First, the state map has been removed, and the operation tracker now provides
status information for any Cid. This implies that the tracker keeps all
operations and operations have a `PhaseDone`. There's also a
new `OperationRemote` type.
Secondly, operations are only created in the tracker and can only be removed
by their creators (they can be overwritten by other operations though).
Operations cannot be accessed directly and modifications are limited to setting
Error for PhaseDone operations.
After created, *Operations are queued in the pinWorker queues which handle any
status updates. This means, that, even when an operation has been removed from
the tracker, status updates will not interfere with any other newer operations.
In the maptracker, only the Unpin worker Cleans operations once processed. A
sucessful unpin is the only way that a delete() happens in the tracker map.
Otherwise, operations stay there until a newer operation for the Cid arrives
and 1) cancels the existing one 2) takes its place. The tracker refuses to
create a new operation if a similar "ongoing" operation of the same type
exists.
The final change is that Recover and RecoverAll() are not async and play by the
same rules as Track() and Untrack(), queueing the items to be recovered.
Note: for stateless pintracker, the tracker will need to Clean() operation
of type OperationPin as well, and complement the Status reported
by the tracker with those coming from IPFS.
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-05-25 16:32:10 +00:00
|
|
|
}
|
2018-08-15 10:27:01 +00:00
|
|
|
ReadyTimeout = 11 * time.Second
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func checkErr(t *testing.T, err error) {
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func randomBytes() []byte {
|
|
|
|
bs := make([]byte, 64, 64)
|
|
|
|
for i := 0; i < len(bs); i++ {
|
|
|
|
b := byte(rand.Int())
|
|
|
|
bs[i] = b
|
|
|
|
}
|
|
|
|
return bs
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
func createComponents(t *testing.T, i int, clusterSecret []byte, staging bool) (host.Host, *Config, *raft.Consensus, []API, IPFSConnector, state.State, PinTracker, PeerMonitor, PinAllocator, Informer, Tracer, *test.IpfsMock) {
|
|
|
|
ctx := context.Background()
|
2017-02-09 15:29:17 +00:00
|
|
|
mock := test.NewIpfsMock()
|
2018-01-16 19:57:54 +00:00
|
|
|
//
|
|
|
|
//clusterAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", clusterPort+i))
|
|
|
|
// Bind on port 0
|
|
|
|
clusterAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
|
|
|
|
//apiAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", apiPort+i))
|
|
|
|
// Bind on port 0
|
|
|
|
apiAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
|
|
|
|
// Bind on Port 0
|
|
|
|
// proxyAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", ipfsProxyPort+i))
|
|
|
|
proxyAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0")
|
2017-02-09 15:29:17 +00:00
|
|
|
nodeAddr, _ := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%d", mock.Addr, mock.Port))
|
2017-01-30 12:12:25 +00:00
|
|
|
priv, pub, err := crypto.GenerateKeyPair(crypto.RSA, 2048)
|
|
|
|
checkErr(t, err)
|
|
|
|
pid, err := peer.IDFromPublicKey(pub)
|
|
|
|
checkErr(t, err)
|
2017-12-01 18:50:13 +00:00
|
|
|
peername := fmt.Sprintf("peer_%d", i)
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
clusterCfg, apiCfg, ipfsproxyCfg, ipfshttpCfg, consensusCfg, maptrackerCfg, statelesstrackerCfg, bmonCfg, psmonCfg, diskInfCfg, tracingCfg := testingConfigs()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
|
|
|
|
clusterCfg.ID = pid
|
2017-12-01 18:50:13 +00:00
|
|
|
clusterCfg.Peername = peername
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
clusterCfg.PrivateKey = priv
|
|
|
|
clusterCfg.Secret = clusterSecret
|
|
|
|
clusterCfg.ListenAddr = clusterAddr
|
|
|
|
clusterCfg.LeaveOnShutdown = false
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
clusterCfg.SetBaseDir("./e2eTestRaft/" + pid.Pretty())
|
|
|
|
|
|
|
|
host, err := NewClusterHost(context.Background(), clusterCfg)
|
|
|
|
checkErr(t, err)
|
|
|
|
|
2018-03-14 22:52:49 +00:00
|
|
|
apiCfg.HTTPListenAddr = apiAddr
|
2018-12-12 18:51:21 +00:00
|
|
|
ipfsproxyCfg.ListenAddr = proxyAddr
|
2018-11-11 10:53:36 +00:00
|
|
|
ipfsproxyCfg.NodeAddr = nodeAddr
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ipfshttpCfg.NodeAddr = nodeAddr
|
|
|
|
consensusCfg.DataFolder = "./e2eTestRaft/" + pid.Pretty()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
api, err := rest.NewAPI(ctx, apiCfg)
|
2017-01-30 12:12:25 +00:00
|
|
|
checkErr(t, err)
|
2018-06-27 04:03:15 +00:00
|
|
|
ipfsProxy, err := rest.NewAPI(ctx, apiCfg)
|
2018-11-11 10:53:36 +00:00
|
|
|
checkErr(t, err)
|
|
|
|
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
ipfs, err := ipfshttp.NewConnector(ipfshttpCfg)
|
2017-01-30 12:12:25 +00:00
|
|
|
checkErr(t, err)
|
2017-02-09 15:29:17 +00:00
|
|
|
state := mapstate.NewMapState()
|
2018-09-12 13:56:10 +00:00
|
|
|
tracker := makePinTracker(t, clusterCfg.ID, maptrackerCfg, statelesstrackerCfg, clusterCfg.Peername)
|
2018-05-08 12:39:45 +00:00
|
|
|
|
|
|
|
mon := makeMonitor(t, host, bmonCfg, psmonCfg)
|
|
|
|
|
2017-09-01 12:09:37 +00:00
|
|
|
alloc := descendalloc.NewAllocator()
|
Issue #162: Rework configuration format
The following commit reimplements ipfs-cluster configuration under
the following premises:
* Each component is initialized with a configuration object
defined by its module
* Each component decides how the JSON representation of its
configuration looks like
* Each component parses and validates its own configuration
* Each component exposes its own defaults
* Component configurations are make the sections of a
central JSON configuration file (which replaces the current
JSON format)
* Component configurations implement a common interface
(config.ComponentConfig) with a set of common operations
* The central configuration file is managed by a
config.ConfigManager which:
* Registers ComponentConfigs
* Assigns the correspondent sections from the JSON file to each
component and delegates the parsing
* Delegates the JSON generation for each section
* Can be notified when the configuration is updated and must be
saved to disk
The new service.json would then look as follows:
```json
{
"cluster": {
"id": "QmTVW8NoRxC5wBhV7WtAYtRn7itipEESfozWN5KmXUQnk2",
"private_key": "<...>",
"secret": "00224102ae6aaf94f2606abf69a0e278251ecc1d64815b617ff19d6d2841f786",
"peers": [],
"bootstrap": [],
"leave_on_shutdown": false,
"listen_multiaddress": "/ip4/0.0.0.0/tcp/9096",
"state_sync_interval": "1m0s",
"ipfs_sync_interval": "2m10s",
"replication_factor": -1,
"monitor_ping_interval": "15s"
},
"consensus": {
"raft": {
"heartbeat_timeout": "1s",
"election_timeout": "1s",
"commit_timeout": "50ms",
"max_append_entries": 64,
"trailing_logs": 10240,
"snapshot_interval": "2m0s",
"snapshot_threshold": 8192,
"leader_lease_timeout": "500ms"
}
},
"api": {
"restapi": {
"listen_multiaddress": "/ip4/127.0.0.1/tcp/9094",
"read_timeout": "30s",
"read_header_timeout": "5s",
"write_timeout": "1m0s",
"idle_timeout": "2m0s"
}
},
"ipfs_connector": {
"ipfshttp": {
"proxy_listen_multiaddress": "/ip4/127.0.0.1/tcp/9095",
"node_multiaddress": "/ip4/127.0.0.1/tcp/5001",
"connect_swarms_delay": "7s",
"proxy_read_timeout": "10m0s",
"proxy_read_header_timeout": "5s",
"proxy_write_timeout": "10m0s",
"proxy_idle_timeout": "1m0s"
}
},
"monitor": {
"monbasic": {
"check_interval": "15s"
}
},
"informer": {
"disk": {
"metric_ttl": "30s",
"metric_type": "freespace"
},
"numpin": {
"metric_ttl": "10s"
}
}
}
```
This new format aims to be easily extensible per component. As such,
it already surfaces quite a few new options which were hardcoded
before.
Additionally, since Go API have changed, some redundant methods have been
removed and small refactoring has happened to take advantage of the new
way.
License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
2017-10-11 18:23:03 +00:00
|
|
|
inf, err := disk.NewInformer(diskInfCfg)
|
|
|
|
checkErr(t, err)
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
raftCon, err := raft.NewConsensus(host, consensusCfg, state, staging)
|
|
|
|
checkErr(t, err)
|
2017-01-30 12:12:25 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
tracer, err := observations.SetupTracing(tracingCfg)
|
|
|
|
checkErr(t, err)
|
|
|
|
|
|
|
|
return host, clusterCfg, raftCon, []API{api, ipfsProxy}, ipfs, state, tracker, mon, alloc, inf, tracer, mock
|
2017-01-30 12:12:25 +00:00
|
|
|
}
|
2016-12-21 13:30:54 +00:00
|
|
|
|
2018-05-08 12:39:45 +00:00
|
|
|
func makeMonitor(t *testing.T, h host.Host, bmonCfg *basic.Config, psmonCfg *pubsubmon.Config) PeerMonitor {
|
|
|
|
var mon PeerMonitor
|
|
|
|
var err error
|
|
|
|
switch pmonitor {
|
|
|
|
case "basic":
|
|
|
|
mon, err = basic.NewMonitor(bmonCfg)
|
|
|
|
case "pubsub":
|
|
|
|
mon, err = pubsubmon.New(h, psmonCfg)
|
|
|
|
default:
|
|
|
|
panic("bad monitor")
|
|
|
|
}
|
|
|
|
checkErr(t, err)
|
|
|
|
return mon
|
|
|
|
}
|
|
|
|
|
2018-09-12 13:56:10 +00:00
|
|
|
func makePinTracker(t *testing.T, pid peer.ID, mptCfg *maptracker.Config, sptCfg *stateless.Config, peerName string) PinTracker {
|
2018-06-07 02:19:15 +00:00
|
|
|
var ptrkr PinTracker
|
|
|
|
switch ptracker {
|
|
|
|
case "map":
|
2018-09-12 13:56:10 +00:00
|
|
|
ptrkr = maptracker.NewMapPinTracker(mptCfg, pid, peerName)
|
2018-06-07 02:19:15 +00:00
|
|
|
case "stateless":
|
2018-09-12 13:56:10 +00:00
|
|
|
ptrkr = stateless.New(sptCfg, pid, peerName)
|
2018-06-07 02:19:15 +00:00
|
|
|
default:
|
|
|
|
panic("bad pintracker")
|
|
|
|
}
|
|
|
|
return ptrkr
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
func createCluster(t *testing.T, host host.Host, clusterCfg *Config, raftCons *raft.Consensus, apis []API, ipfs IPFSConnector, state state.State, tracker PinTracker, mon PeerMonitor, alloc PinAllocator, inf Informer, tracer Tracer) *Cluster {
|
|
|
|
cl, err := NewCluster(host, clusterCfg, raftCons, apis, ipfs, state, tracker, mon, alloc, inf, tracer)
|
2017-01-30 12:12:25 +00:00
|
|
|
checkErr(t, err)
|
|
|
|
return cl
|
|
|
|
}
|
|
|
|
|
2017-07-04 20:39:10 +00:00
|
|
|
func createOnePeerCluster(t *testing.T, nth int, clusterSecret []byte) (*Cluster, *test.IpfsMock) {
|
2018-06-27 04:03:15 +00:00
|
|
|
host, clusterCfg, consensusCfg, api, ipfs, state, tracker, mon, alloc, inf, tracer, mock := createComponents(t, nth, clusterSecret, false)
|
|
|
|
cl := createCluster(t, host, clusterCfg, consensusCfg, api, ipfs, state, tracker, mon, alloc, inf, tracer)
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
<-cl.Ready()
|
2017-01-30 12:12:25 +00:00
|
|
|
return cl, mock
|
|
|
|
}
|
2016-12-21 13:30:54 +00:00
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
func createClusters(t *testing.T) ([]*Cluster, []*test.IpfsMock) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-01-30 12:12:25 +00:00
|
|
|
os.RemoveAll("./e2eTestRaft")
|
|
|
|
cfgs := make([]*Config, nClusters, nClusters)
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
raftCons := make([]*raft.Consensus, nClusters, nClusters)
|
2018-11-11 10:53:36 +00:00
|
|
|
apis := make([][]API, nClusters, nClusters)
|
2017-02-13 15:46:53 +00:00
|
|
|
ipfss := make([]IPFSConnector, nClusters, nClusters)
|
2017-03-10 16:24:25 +00:00
|
|
|
states := make([]state.State, nClusters, nClusters)
|
2017-02-13 15:46:53 +00:00
|
|
|
trackers := make([]PinTracker, nClusters, nClusters)
|
|
|
|
mons := make([]PeerMonitor, nClusters, nClusters)
|
|
|
|
allocs := make([]PinAllocator, nClusters, nClusters)
|
|
|
|
infs := make([]Informer, nClusters, nClusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
tracers := make([]Tracer, nClusters, nClusters)
|
2017-02-09 15:29:17 +00:00
|
|
|
ipfsMocks := make([]*test.IpfsMock, nClusters, nClusters)
|
2018-03-29 18:18:15 +00:00
|
|
|
|
|
|
|
hosts := make([]host.Host, nClusters, nClusters)
|
2017-01-30 12:12:25 +00:00
|
|
|
clusters := make([]*Cluster, nClusters, nClusters)
|
|
|
|
|
2018-01-25 22:34:06 +00:00
|
|
|
// Uncomment when testing with fixed ports
|
|
|
|
// clusterPeers := make([]ma.Multiaddr, nClusters, nClusters)
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
for i := 0; i < nClusters; i++ {
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
// staging = true for all except first (i==0)
|
2018-06-27 04:03:15 +00:00
|
|
|
hosts[i], cfgs[i], raftCons[i], apis[i], ipfss[i], states[i], trackers[i], mons[i], allocs[i], infs[i], tracers[i], ipfsMocks[i] = createComponents(t, i, testingClusterSecret, i != 0)
|
2018-03-29 18:18:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// open connections among all hosts
|
|
|
|
for _, h := range hosts {
|
|
|
|
for _, h2 := range hosts {
|
|
|
|
if h.ID() != h2.ID() {
|
|
|
|
|
|
|
|
h.Peerstore().AddAddrs(h2.ID(), h2.Addrs(), peerstore.PermanentAddrTTL)
|
|
|
|
_, err := h.Network().DialPeer(context.Background(), h2.ID())
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-16 19:57:54 +00:00
|
|
|
// Start first node
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0] = createCluster(t, hosts[0], cfgs[0], raftCons[0], apis[0], ipfss[0], states[0], trackers[0], mons[0], allocs[0], infs[0], tracers[0])
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
<-clusters[0].Ready()
|
2018-10-25 12:55:01 +00:00
|
|
|
bootstrapAddr := clusterAddr(clusters[0])
|
2018-01-16 19:57:54 +00:00
|
|
|
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
// Start the rest and join
|
2018-01-16 19:57:54 +00:00
|
|
|
for i := 1; i < nClusters; i++ {
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[i] = createCluster(t, hosts[i], cfgs[i], raftCons[i], apis[i], ipfss[i], states[i], trackers[i], mons[i], allocs[i], infs[i], tracers[i])
|
|
|
|
err := clusters[i].Join(ctx, bootstrapAddr)
|
Feat: emancipate Consensus from the Cluster component
This commit promotes the Consensus component (and Raft) to become a fully
independent thing like other components, passed to NewCluster during
initialization. Cluster (main component) no longer creates the consensus
layer internally. This has triggered a number of breaking changes
that I will explain below.
Motivation: Future work will require the possibility of running Cluster
with a consensus layer that is not Raft. The "consensus" layer is in charge
of maintaining two things:
* The current cluster peerset, as required by the implementation
* The current cluster pinset (shared state)
While the pinset maintenance has always been in the consensus layer, the
peerset maintenance was handled by the main component (starting by the "peers"
key in the configuration) AND the Raft component (internally)
and this generated lots of confusion: if the user edited the peers in the
configuration they would be greeted with an error.
The bootstrap process (adding a peer to an existing cluster) and configuration
key also complicated many things, since the main component did it, but only
when the consensus was initialized and in single peer mode.
In all this we also mixed the peerstore (list of peer addresses in the libp2p
host) with the peerset, when they need not to be linked.
By initializing the consensus layer before calling NewCluster, all the
difficulties in maintaining the current implementation in the same way
have come to light. Thus, the following changes have been introduced:
* Remove "peers" and "bootstrap" keys from the configuration: we no longer
edit or save the configuration files. This was a very bad practice, requiring
write permissions by the process to the file containing the private key and
additionally made things like Puppet deployments of cluster difficult as
configuration would mutate from its initial version. Needless to say all the
maintenance associated to making sure peers and bootstrap had correct values
when peers are bootstrapped or removed. A loud and detailed error message has
been added when staring cluster with an old config, along with instructions on
how to move forward.
* Introduce a PeerstoreFile ("peerstore") which stores peer addresses: in
ipfs, the peerstore is not persisted because it can be re-built from the
network bootstrappers and the DHT. Cluster should probably also allow
discoverability of peers addresses (when not bootstrapping, as in that case
we have it), but in the meantime, we will read and persist the peerstore
addresses for cluster peers in this file, different from the configuration.
Note that dns multiaddresses are now fully supported and no IPs are saved
when we have DNS multiaddresses for a peer.
* The former "peer_manager" code is now a pstoremgr module, providing utilities
to parse, add, list and generally maintain the libp2p host peerstore, including
operations on the PeerstoreFile. This "pstoremgr" can now also be extended to
perform address autodiscovery and other things indepedently from Cluster.
* Create and initialize Raft outside of the main Cluster component: since we
can now launch Raft independently from Cluster, we have more degrees of
freedom. A new "staging" option when creating the object allows a raft peer to
be launched in Staging mode, waiting to be added to a running consensus, and
thus, not electing itself as leader or doing anything like we were doing
before. This additionally allows us to track when the peer has become a
Voter, which only happens when it's caught up with the state, something that
was wonky previously.
* The raft configuration now includes an InitPeerset key, which allows to
provide a peerset for new peers and which is ignored when staging==true. The
whole Raft initialization code is way cleaner and stronger now.
* Cluster peer bootsrapping is now an ipfs-cluster-service feature. The
--bootstrap flag works as before (additionally allowing comma-separated-list
of entries). What bootstrap does, is to initialize Raft with staging == true,
and then call Join in the main cluster component. Only when the Raft peer
transitions to Voter, consensus becomes ready, and cluster becomes Ready.
This is cleaner, works better and is less complex than before (supporting
both flags and config values). We also backup and clean the state whenever
we are boostrapping, automatically
* ipfs-cluster-service no longer runs the daemon. Starting cluster needs
now "ipfs-cluster-service daemon". The daemon specific flags (bootstrap,
alloc) are now flags for the daemon subcommand. Here we mimic ipfs ("ipfs"
does not start the daemon but print help) and pave the path for merging both
service and ctl in the future.
While this brings some breaking changes, it significantly reduces the
complexity of the configuration, the code and most importantly, the
documentation. It should be easier now to explain the user what is the
right way to launch a cluster peer, and more difficult to make mistakes.
As a side effect, the PR also:
* Fixes #381 - peers with dynamic addresses
* Fixes #371 - peers should be Raft configuration option
* Fixes #378 - waitForUpdates may return before state fully synced
* Fixes #235 - config option shadowing (no cfg saves, no need to shadow)
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-04-28 22:22:23 +00:00
|
|
|
if err != nil {
|
|
|
|
logger.Error(err)
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
<-clusters[i].Ready()
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
2018-03-29 18:18:15 +00:00
|
|
|
waitForLeader(t, clusters)
|
2017-02-02 22:52:06 +00:00
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
return clusters, ipfsMocks
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
func shutdownClusters(t *testing.T, clusters []*Cluster, m []*test.IpfsMock) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
for i, c := range clusters {
|
2018-06-27 04:03:15 +00:00
|
|
|
err := c.Shutdown(ctx)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
2017-11-29 16:49:03 +00:00
|
|
|
m[i].Close()
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
os.RemoveAll("./e2eTestRaft")
|
|
|
|
}
|
|
|
|
|
|
|
|
func runF(t *testing.T, clusters []*Cluster, f func(*testing.T, *Cluster)) {
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
for _, c := range clusters {
|
|
|
|
wg.Add(1)
|
2016-12-21 19:46:00 +00:00
|
|
|
go func(c *Cluster) {
|
2016-12-21 13:30:54 +00:00
|
|
|
defer wg.Done()
|
|
|
|
f(t, c)
|
2016-12-21 19:46:00 +00:00
|
|
|
}(c)
|
2016-12-21 13:30:54 +00:00
|
|
|
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
}
|
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
//////////////////////////////////////
|
|
|
|
// Delay and wait functions
|
|
|
|
//
|
|
|
|
// Delays are used in tests to wait for certain events to happen:
|
|
|
|
// * ttlDelay() waits for metrics to arrive. If you pin something
|
|
|
|
// and your next operation depends on updated metrics, you need to wait
|
|
|
|
// * pinDelay() accounts for the time necessary to pin something and for the new
|
|
|
|
// log entry to be visible in all cluster peers
|
|
|
|
// * delay just sleeps a second or two.
|
|
|
|
// * waitForLeader functions make sure there is a raft leader, for example,
|
|
|
|
// after killing the leader.
|
|
|
|
//
|
|
|
|
// The values for delays are a result of testing and adjusting so tests pass
|
|
|
|
// in travis, jenkins etc., taking into account the values used in the
|
|
|
|
// testing configuration (config_test.go).
|
2016-12-21 13:30:54 +00:00
|
|
|
func delay() {
|
2017-02-02 22:52:06 +00:00
|
|
|
var d int
|
|
|
|
if nClusters > 10 {
|
2018-05-08 12:39:45 +00:00
|
|
|
d = 3000
|
2017-02-02 22:52:06 +00:00
|
|
|
} else {
|
2018-05-08 12:39:45 +00:00
|
|
|
d = 2000
|
2017-02-02 22:52:06 +00:00
|
|
|
}
|
2018-03-16 16:37:39 +00:00
|
|
|
time.Sleep(time.Duration(d) * time.Millisecond)
|
|
|
|
}
|
|
|
|
|
|
|
|
func pinDelay() {
|
2018-05-08 12:39:45 +00:00
|
|
|
time.Sleep(500 * time.Millisecond)
|
2018-03-16 16:37:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func ttlDelay() {
|
|
|
|
diskInfCfg := &disk.Config{}
|
|
|
|
diskInfCfg.LoadJSON(testingDiskInfCfg)
|
|
|
|
time.Sleep(diskInfCfg.MetricTTL * 3)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
// Like waitForLeader but letting metrics expire before waiting, and
|
|
|
|
// waiting for new metrics to arrive afterwards.
|
|
|
|
func waitForLeaderAndMetrics(t *testing.T, clusters []*Cluster) {
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeader(t, clusters)
|
2018-03-29 18:18:15 +00:00
|
|
|
ttlDelay()
|
|
|
|
}
|
2017-03-09 13:44:14 +00:00
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
// Makes sure there is a leader and everyone knows about it.
|
|
|
|
func waitForLeader(t *testing.T, clusters []*Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-03-16 16:37:39 +00:00
|
|
|
timer := time.NewTimer(time.Minute)
|
2018-03-29 18:18:15 +00:00
|
|
|
ticker := time.NewTicker(100 * time.Millisecond)
|
2017-03-09 13:44:14 +00:00
|
|
|
|
|
|
|
loop:
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-timer.C:
|
|
|
|
t.Fatal("timed out waiting for a leader")
|
|
|
|
case <-ticker.C:
|
2018-03-16 16:37:39 +00:00
|
|
|
for _, cl := range clusters {
|
|
|
|
if cl.shutdownB {
|
|
|
|
continue // skip shutdown clusters
|
|
|
|
}
|
2018-06-27 04:03:15 +00:00
|
|
|
_, err := cl.consensus.Leader(ctx)
|
2018-03-16 16:37:39 +00:00
|
|
|
if err != nil {
|
|
|
|
continue loop
|
|
|
|
}
|
2017-03-09 13:44:14 +00:00
|
|
|
}
|
2018-03-16 16:37:39 +00:00
|
|
|
break loop
|
2017-03-09 13:44:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
/////////////////////////////////////////
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
func TestClustersVersion(t *testing.T) {
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
|
|
v := c.Version()
|
2018-12-18 14:44:11 +00:00
|
|
|
if v != version.Version.String() {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Error("Bad version")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
func TestClustersPeers(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-01-26 18:59:31 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
|
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer
|
2018-06-27 04:03:15 +00:00
|
|
|
peers := clusters[j].Peers(ctx)
|
2018-01-16 19:57:54 +00:00
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
if len(peers) != nClusters {
|
|
|
|
t.Fatal("expected as many peers as clusters")
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
clusterIDMap := make(map[peer.ID]api.ID)
|
|
|
|
peerIDMap := make(map[peer.ID]api.ID)
|
2017-01-26 18:59:31 +00:00
|
|
|
|
|
|
|
for _, c := range clusters {
|
2018-06-27 04:03:15 +00:00
|
|
|
id := c.ID(ctx)
|
2017-01-26 18:59:31 +00:00
|
|
|
clusterIDMap[id.ID] = id
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, p := range peers {
|
|
|
|
peerIDMap[p.ID] = p
|
|
|
|
}
|
|
|
|
|
|
|
|
for k, id := range clusterIDMap {
|
|
|
|
id2, ok := peerIDMap[k]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("expected id in both maps")
|
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
//if !crypto.KeyEqual(id.PublicKey, id2.PublicKey) {
|
|
|
|
// t.Error("expected same public key")
|
|
|
|
//}
|
2017-01-26 18:59:31 +00:00
|
|
|
if id.IPFS.ID != id2.IPFS.ID {
|
|
|
|
t.Error("expected same ipfs daemon ID")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
func TestClustersPin(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
exampleCid, _ := cid.Decode(test.TestCid1)
|
2016-12-21 13:30:54 +00:00
|
|
|
prefix := exampleCid.Prefix()
|
2018-03-16 16:37:39 +00:00
|
|
|
|
|
|
|
ttlDelay()
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
for i := 0; i < nPins; i++ {
|
2017-01-26 18:59:31 +00:00
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer
|
2016-12-21 13:30:54 +00:00
|
|
|
h, err := prefix.Sum(randomBytes()) // create random cid
|
|
|
|
checkErr(t, err)
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[j].Pin(ctx, api.PinCid(h))
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Errorf("error pinning %s: %s", h, err)
|
|
|
|
}
|
|
|
|
// Test re-pin
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[j].Pin(ctx, api.PinCid(h))
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Errorf("error repinning %s: %s", h, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
delay()
|
|
|
|
fpinned := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
status := c.tracker.StatusAll(ctx)
|
2016-12-21 13:30:54 +00:00
|
|
|
for _, v := range status {
|
2017-02-08 17:04:08 +00:00
|
|
|
if v.Status != api.TrackerStatusPinned {
|
2018-05-11 17:38:40 +00:00
|
|
|
t.Errorf("%s should have been pinned but it is %s", v.Cid, v.Status)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if l := len(status); l != nPins {
|
|
|
|
t.Errorf("Pinned %d out of %d requests", l, nPins)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
runF(t, clusters, fpinned)
|
|
|
|
|
|
|
|
// Unpin everything
|
2018-06-27 04:03:15 +00:00
|
|
|
pinList := clusters[0].Pins(ctx)
|
2016-12-21 13:30:54 +00:00
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
for i := 0; i < len(pinList); i++ {
|
2018-03-18 19:29:02 +00:00
|
|
|
// test re-unpin fails
|
2017-01-26 18:59:31 +00:00
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[j].Unpin(ctx, pinList[i].Cid)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
2017-02-15 14:16:34 +00:00
|
|
|
t.Errorf("error unpinning %s: %s", pinList[i].Cid, err)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
2018-03-18 19:29:02 +00:00
|
|
|
}
|
|
|
|
delay()
|
|
|
|
for i := 0; i < nPins; i++ {
|
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[j].Unpin(ctx, pinList[i].Cid)
|
2018-03-18 19:29:02 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Errorf("expected error re-unpinning %s: %s", pinList[i].Cid, err)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
2018-03-18 19:29:02 +00:00
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
delay()
|
|
|
|
funpinned := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
status := c.tracker.StatusAll(ctx)
|
2018-05-11 17:38:40 +00:00
|
|
|
for _, v := range status {
|
|
|
|
t.Errorf("%s should have been unpinned but it is %s", v.Cid, v.Status)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
runF(t, clusters, funpinned)
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
func TestClustersStatusAll(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2016-12-21 13:30:54 +00:00
|
|
|
// Global status
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
statuses, err := c.StatusAll(ctx)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
2017-06-21 12:16:28 +00:00
|
|
|
if len(statuses) != 1 {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Fatal("bad status. Expected one item")
|
|
|
|
}
|
2017-02-09 15:29:17 +00:00
|
|
|
if statuses[0].Cid.String() != test.TestCid1 {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Error("bad cid in status")
|
|
|
|
}
|
2017-01-25 17:07:19 +00:00
|
|
|
info := statuses[0].PeerMap
|
2016-12-21 13:30:54 +00:00
|
|
|
if len(info) != nClusters {
|
|
|
|
t.Error("bad info in status")
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
if info[c.host.ID()].Status != api.TrackerStatusPinned {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Error("the hash should have been pinned")
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
status, err := c.Status(ctx, h)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
|
|
|
|
2017-01-25 17:07:19 +00:00
|
|
|
pinfo, ok := status.PeerMap[c.host.ID()]
|
2016-12-21 13:30:54 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatal("Host not in status")
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
if pinfo.Status != api.TrackerStatusPinned {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Error("the status should show the hash as pinned")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2017-06-21 12:16:28 +00:00
|
|
|
func TestClustersStatusAllWithErrors(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-06-21 12:16:28 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-06-21 12:16:28 +00:00
|
|
|
|
|
|
|
// shutdown 1 cluster peer
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[1].Shutdown(ctx)
|
2018-03-16 16:37:39 +00:00
|
|
|
delay()
|
2017-06-21 12:16:28 +00:00
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
|
|
// skip if it's the shutdown peer
|
2018-06-27 04:03:15 +00:00
|
|
|
if c.ID(ctx).ID == clusters[1].ID(ctx).ID {
|
2017-06-21 12:16:28 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
statuses, err := c.StatusAll(ctx)
|
2017-06-21 12:16:28 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
|
|
|
if len(statuses) != 1 {
|
|
|
|
t.Fatal("bad status. Expected one item")
|
|
|
|
}
|
|
|
|
|
|
|
|
stts := statuses[0]
|
|
|
|
if len(stts.PeerMap) != nClusters {
|
|
|
|
t.Error("bad number of peers in status")
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
errst := stts.PeerMap[clusters[1].ID(ctx).ID]
|
2017-06-21 12:16:28 +00:00
|
|
|
|
|
|
|
if errst.Cid.String() != test.TestCid1 {
|
|
|
|
t.Error("errored pinInfo should have a good cid")
|
|
|
|
}
|
|
|
|
|
|
|
|
if errst.Status != api.TrackerStatusClusterError {
|
|
|
|
t.Error("erroring status should be set to ClusterError")
|
|
|
|
}
|
|
|
|
|
|
|
|
// now check with Cid status
|
2018-06-27 04:03:15 +00:00
|
|
|
status, err := c.Status(ctx, h)
|
2017-06-21 12:16:28 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfo := status.PeerMap[clusters[1].ID(ctx).ID]
|
2017-06-21 12:16:28 +00:00
|
|
|
|
|
|
|
if pinfo.Status != api.TrackerStatusClusterError {
|
|
|
|
t.Error("erroring status should be ClusterError")
|
|
|
|
}
|
|
|
|
|
|
|
|
if pinfo.Cid.String() != test.TestCid1 {
|
|
|
|
t.Error("errored status should have a good cid")
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
func TestClustersSyncAllLocal(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.ErrorCid) // This cid always fails
|
|
|
|
h2, _ := cid.Decode(test.TestCid2)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
|
|
|
clusters[0].Pin(ctx, api.PinCid(h2))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
pinDelay()
|
|
|
|
|
2016-12-21 13:30:54 +00:00
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
|
|
// Sync bad ID
|
2018-06-27 04:03:15 +00:00
|
|
|
infos, err := c.SyncAllLocal(ctx)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
// LocalSync() is asynchronous and should not show an
|
|
|
|
// error even if Recover() fails.
|
|
|
|
t.Error(err)
|
|
|
|
}
|
|
|
|
if len(infos) != 1 {
|
2018-06-07 02:19:15 +00:00
|
|
|
t.Fatalf("expected 1 elem slice, got = %d", len(infos))
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
// Last-known state may still be pinning
|
2017-02-08 17:04:08 +00:00
|
|
|
if infos[0].Status != api.TrackerStatusPinError && infos[0].Status != api.TrackerStatusPinning {
|
2018-06-07 02:19:15 +00:00
|
|
|
t.Errorf("element should be in Pinning or PinError state, got = %v", infos[0].Status)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
// Test Local syncs
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
func TestClustersSyncLocal(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.ErrorCid) // This cid always fails
|
|
|
|
h2, _ := cid.Decode(test.TestCid2)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
|
|
|
clusters[0].Pin(ctx, api.PinCid(h2))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
pinDelay()
|
2016-12-21 13:30:54 +00:00
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
info, err := c.SyncLocal(ctx, h)
|
2017-01-25 17:07:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
if info.Status != api.TrackerStatusPinError && info.Status != api.TrackerStatusPinning {
|
2017-01-25 17:07:19 +00:00
|
|
|
t.Errorf("element is %s and not PinError", info.Status)
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Sync good ID
|
2018-06-27 04:03:15 +00:00
|
|
|
info, err = c.SyncLocal(ctx, h2)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
if info.Status != api.TrackerStatusPinned {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Error("element should be in Pinned state")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Test Local syncs
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
func TestClustersSyncAll(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.ErrorCid) // This cid always fails
|
|
|
|
h2, _ := cid.Decode(test.TestCid2)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
|
|
|
clusters[0].Pin(ctx, api.PinCid(h2))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
pinDelay()
|
2016-12-21 13:30:54 +00:00
|
|
|
|
2017-01-26 18:59:31 +00:00
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer
|
2018-06-27 04:03:15 +00:00
|
|
|
ginfos, err := clusters[j].SyncAll(ctx)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if len(ginfos) != 1 {
|
2018-06-07 02:19:15 +00:00
|
|
|
t.Fatalf("expected globalsync to have 1 elements, got = %d", len(ginfos))
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
2017-02-09 15:29:17 +00:00
|
|
|
if ginfos[0].Cid.String() != test.ErrorCid {
|
|
|
|
t.Error("expected globalsync to have problems with test.ErrorCid")
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
for _, c := range clusters {
|
2017-01-25 17:07:19 +00:00
|
|
|
inf, ok := ginfos[0].PeerMap[c.host.ID()]
|
2016-12-21 13:30:54 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatal("GlobalPinInfo should have this cluster")
|
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
if inf.Status != api.TrackerStatusPinError && inf.Status != api.TrackerStatusPinning {
|
2017-01-26 18:59:31 +00:00
|
|
|
t.Error("should be PinError in all peers")
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
func TestClustersSync(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2016-12-21 13:30:54 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.ErrorCid) // This cid always fails
|
|
|
|
h2, _ := cid.Decode(test.TestCid2)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
|
|
|
clusters[0].Pin(ctx, api.PinCid(h2))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
pinDelay()
|
2016-12-21 13:30:54 +00:00
|
|
|
|
|
|
|
j := rand.Intn(nClusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
ginfo, err := clusters[j].Sync(ctx, h)
|
2017-01-23 23:52:42 +00:00
|
|
|
if err != nil {
|
|
|
|
// we always attempt to return a valid response
|
|
|
|
// with errors contained in GlobalPinInfo
|
|
|
|
t.Fatal("did not expect an error")
|
|
|
|
}
|
2017-01-25 17:07:19 +00:00
|
|
|
pinfo, ok := ginfo.PeerMap[clusters[j].host.ID()]
|
2017-01-23 23:52:42 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatal("should have info for this host")
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
2017-01-23 23:52:42 +00:00
|
|
|
if pinfo.Error == "" {
|
|
|
|
t.Error("pinInfo error should not be empty")
|
|
|
|
}
|
|
|
|
|
2017-02-09 15:29:17 +00:00
|
|
|
if ginfo.Cid.String() != test.ErrorCid {
|
|
|
|
t.Error("GlobalPinInfo should be for test.ErrorCid")
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range clusters {
|
2017-01-25 17:07:19 +00:00
|
|
|
inf, ok := ginfo.PeerMap[c.host.ID()]
|
2016-12-21 13:30:54 +00:00
|
|
|
if !ok {
|
2016-12-23 18:35:37 +00:00
|
|
|
t.Logf("%+v", ginfo)
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Fatal("GlobalPinInfo should not be empty for this host")
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
if inf.Status != api.TrackerStatusPinError && inf.Status != api.TrackerStatusPinning {
|
2017-01-26 18:59:31 +00:00
|
|
|
t.Error("should be PinError or Pinning in all peers")
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test with a good Cid
|
|
|
|
j = rand.Intn(nClusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
ginfo, err = clusters[j].Sync(ctx, h2)
|
2017-01-25 18:38:23 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2017-02-09 15:29:17 +00:00
|
|
|
if ginfo.Cid.String() != test.TestCid2 {
|
2017-01-25 18:38:23 +00:00
|
|
|
t.Error("GlobalPinInfo should be for testrCid2")
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range clusters {
|
|
|
|
inf, ok := ginfo.PeerMap[c.host.ID()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("GlobalPinInfo should have this cluster")
|
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
if inf.Status != api.TrackerStatusPinned {
|
2017-01-26 18:59:31 +00:00
|
|
|
t.Error("the GlobalPinInfo should show Pinned in all peers")
|
2017-01-25 18:38:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestClustersRecoverLocal(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-01-25 18:38:23 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.ErrorCid) // This cid always fails
|
|
|
|
h2, _ := cid.Decode(test.TestCid2)
|
2018-03-16 16:37:39 +00:00
|
|
|
|
|
|
|
ttlDelay()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
|
|
|
clusters[0].Pin(ctx, api.PinCid(h2))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
pinDelay()
|
2017-01-25 18:38:23 +00:00
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
info, err := c.RecoverLocal(ctx, h)
|
Fix: maptracker race issues
This commit attempts to fix race issues in the maptracker since the
introduction of the OperationTracker.
There were two main problems:
* Duplicity tracking the state both in the state map and the opTracker
* Non atomiciy of operations with different threads being able to affect
other threads operations.
A test performing random Track/Untracks on the same Cid quickly showed
that items would sometimes stay as pin_queued or pin_unqueued. That happened
because operations could be cancelled under the hood by a different request,
while leaving the map status untouched.
It was not simply to deal with this issues without a refactoring.
First, the state map has been removed, and the operation tracker now provides
status information for any Cid. This implies that the tracker keeps all
operations and operations have a `PhaseDone`. There's also a
new `OperationRemote` type.
Secondly, operations are only created in the tracker and can only be removed
by their creators (they can be overwritten by other operations though).
Operations cannot be accessed directly and modifications are limited to setting
Error for PhaseDone operations.
After created, *Operations are queued in the pinWorker queues which handle any
status updates. This means, that, even when an operation has been removed from
the tracker, status updates will not interfere with any other newer operations.
In the maptracker, only the Unpin worker Cleans operations once processed. A
sucessful unpin is the only way that a delete() happens in the tracker map.
Otherwise, operations stay there until a newer operation for the Cid arrives
and 1) cancels the existing one 2) takes its place. The tracker refuses to
create a new operation if a similar "ongoing" operation of the same type
exists.
The final change is that Recover and RecoverAll() are not async and play by the
same rules as Track() and Untrack(), queueing the items to be recovered.
Note: for stateless pintracker, the tracker will need to Clean() operation
of type OperationPin as well, and complement the Status reported
by the tracker with those coming from IPFS.
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-05-25 16:32:10 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
2017-01-25 18:38:23 +00:00
|
|
|
}
|
Fix: maptracker race issues
This commit attempts to fix race issues in the maptracker since the
introduction of the OperationTracker.
There were two main problems:
* Duplicity tracking the state both in the state map and the opTracker
* Non atomiciy of operations with different threads being able to affect
other threads operations.
A test performing random Track/Untracks on the same Cid quickly showed
that items would sometimes stay as pin_queued or pin_unqueued. That happened
because operations could be cancelled under the hood by a different request,
while leaving the map status untouched.
It was not simply to deal with this issues without a refactoring.
First, the state map has been removed, and the operation tracker now provides
status information for any Cid. This implies that the tracker keeps all
operations and operations have a `PhaseDone`. There's also a
new `OperationRemote` type.
Secondly, operations are only created in the tracker and can only be removed
by their creators (they can be overwritten by other operations though).
Operations cannot be accessed directly and modifications are limited to setting
Error for PhaseDone operations.
After created, *Operations are queued in the pinWorker queues which handle any
status updates. This means, that, even when an operation has been removed from
the tracker, status updates will not interfere with any other newer operations.
In the maptracker, only the Unpin worker Cleans operations once processed. A
sucessful unpin is the only way that a delete() happens in the tracker map.
Otherwise, operations stay there until a newer operation for the Cid arrives
and 1) cancels the existing one 2) takes its place. The tracker refuses to
create a new operation if a similar "ongoing" operation of the same type
exists.
The final change is that Recover and RecoverAll() are not async and play by the
same rules as Track() and Untrack(), queueing the items to be recovered.
Note: for stateless pintracker, the tracker will need to Clean() operation
of type OperationPin as well, and complement the Status reported
by the tracker with those coming from IPFS.
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-05-25 16:32:10 +00:00
|
|
|
// Wait for queue to be processed
|
|
|
|
delay()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
info = c.StatusLocal(ctx, h)
|
2017-02-08 17:04:08 +00:00
|
|
|
if info.Status != api.TrackerStatusPinError {
|
2017-01-25 18:38:23 +00:00
|
|
|
t.Errorf("element is %s and not PinError", info.Status)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Recover good ID
|
2018-06-27 04:03:15 +00:00
|
|
|
info, err = c.SyncLocal(ctx, h2)
|
2017-01-25 18:38:23 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
if info.Status != api.TrackerStatusPinned {
|
2017-01-25 18:38:23 +00:00
|
|
|
t.Error("element should be in Pinned state")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Test Local syncs
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestClustersRecover(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-01-25 18:38:23 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
2017-02-09 15:29:17 +00:00
|
|
|
h, _ := cid.Decode(test.ErrorCid) // This cid always fails
|
|
|
|
h2, _ := cid.Decode(test.TestCid2)
|
2018-03-16 16:37:39 +00:00
|
|
|
|
|
|
|
ttlDelay()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
|
|
|
clusters[0].Pin(ctx, api.PinCid(h2))
|
2017-01-25 18:38:23 +00:00
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
pinDelay()
|
2017-01-25 18:38:23 +00:00
|
|
|
|
|
|
|
j := rand.Intn(nClusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
_, err := clusters[j].Recover(ctx, h)
|
2017-01-25 18:38:23 +00:00
|
|
|
if err != nil {
|
|
|
|
// we always attempt to return a valid response
|
|
|
|
// with errors contained in GlobalPinInfo
|
|
|
|
t.Fatal("did not expect an error")
|
|
|
|
}
|
Fix: maptracker race issues
This commit attempts to fix race issues in the maptracker since the
introduction of the OperationTracker.
There were two main problems:
* Duplicity tracking the state both in the state map and the opTracker
* Non atomiciy of operations with different threads being able to affect
other threads operations.
A test performing random Track/Untracks on the same Cid quickly showed
that items would sometimes stay as pin_queued or pin_unqueued. That happened
because operations could be cancelled under the hood by a different request,
while leaving the map status untouched.
It was not simply to deal with this issues without a refactoring.
First, the state map has been removed, and the operation tracker now provides
status information for any Cid. This implies that the tracker keeps all
operations and operations have a `PhaseDone`. There's also a
new `OperationRemote` type.
Secondly, operations are only created in the tracker and can only be removed
by their creators (they can be overwritten by other operations though).
Operations cannot be accessed directly and modifications are limited to setting
Error for PhaseDone operations.
After created, *Operations are queued in the pinWorker queues which handle any
status updates. This means, that, even when an operation has been removed from
the tracker, status updates will not interfere with any other newer operations.
In the maptracker, only the Unpin worker Cleans operations once processed. A
sucessful unpin is the only way that a delete() happens in the tracker map.
Otherwise, operations stay there until a newer operation for the Cid arrives
and 1) cancels the existing one 2) takes its place. The tracker refuses to
create a new operation if a similar "ongoing" operation of the same type
exists.
The final change is that Recover and RecoverAll() are not async and play by the
same rules as Track() and Untrack(), queueing the items to be recovered.
Note: for stateless pintracker, the tracker will need to Clean() operation
of type OperationPin as well, and complement the Status reported
by the tracker with those coming from IPFS.
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-05-25 16:32:10 +00:00
|
|
|
|
|
|
|
// Wait for queue to be processed
|
|
|
|
delay()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
ginfo, err := clusters[j].Status(ctx, h)
|
Fix: maptracker race issues
This commit attempts to fix race issues in the maptracker since the
introduction of the OperationTracker.
There were two main problems:
* Duplicity tracking the state both in the state map and the opTracker
* Non atomiciy of operations with different threads being able to affect
other threads operations.
A test performing random Track/Untracks on the same Cid quickly showed
that items would sometimes stay as pin_queued or pin_unqueued. That happened
because operations could be cancelled under the hood by a different request,
while leaving the map status untouched.
It was not simply to deal with this issues without a refactoring.
First, the state map has been removed, and the operation tracker now provides
status information for any Cid. This implies that the tracker keeps all
operations and operations have a `PhaseDone`. There's also a
new `OperationRemote` type.
Secondly, operations are only created in the tracker and can only be removed
by their creators (they can be overwritten by other operations though).
Operations cannot be accessed directly and modifications are limited to setting
Error for PhaseDone operations.
After created, *Operations are queued in the pinWorker queues which handle any
status updates. This means, that, even when an operation has been removed from
the tracker, status updates will not interfere with any other newer operations.
In the maptracker, only the Unpin worker Cleans operations once processed. A
sucessful unpin is the only way that a delete() happens in the tracker map.
Otherwise, operations stay there until a newer operation for the Cid arrives
and 1) cancels the existing one 2) takes its place. The tracker refuses to
create a new operation if a similar "ongoing" operation of the same type
exists.
The final change is that Recover and RecoverAll() are not async and play by the
same rules as Track() and Untrack(), queueing the items to be recovered.
Note: for stateless pintracker, the tracker will need to Clean() operation
of type OperationPin as well, and complement the Status reported
by the tracker with those coming from IPFS.
License: MIT
Signed-off-by: Hector Sanjuan <code@hector.link>
2018-05-25 16:32:10 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2017-01-25 18:38:23 +00:00
|
|
|
pinfo, ok := ginfo.PeerMap[clusters[j].host.ID()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("should have info for this host")
|
|
|
|
}
|
|
|
|
if pinfo.Error == "" {
|
|
|
|
t.Error("pinInfo error should not be empty")
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range clusters {
|
|
|
|
inf, ok := ginfo.PeerMap[c.host.ID()]
|
|
|
|
if !ok {
|
|
|
|
t.Fatal("GlobalPinInfo should not be empty for this host")
|
|
|
|
}
|
|
|
|
|
2017-02-08 17:04:08 +00:00
|
|
|
if inf.Status != api.TrackerStatusPinError {
|
|
|
|
t.Logf("%+v", inf)
|
2017-01-26 18:59:31 +00:00
|
|
|
t.Error("should be PinError in all peers")
|
2017-01-25 18:38:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Test with a good Cid
|
|
|
|
j = rand.Intn(nClusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
ginfo, err = clusters[j].Recover(ctx, h2)
|
2016-12-21 13:30:54 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2017-02-09 15:29:17 +00:00
|
|
|
if ginfo.Cid.String() != test.TestCid2 {
|
2016-12-21 13:30:54 +00:00
|
|
|
t.Error("GlobalPinInfo should be for testrCid2")
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, c := range clusters {
|
2017-01-25 17:07:19 +00:00
|
|
|
inf, ok := ginfo.PeerMap[c.host.ID()]
|
2016-12-21 13:30:54 +00:00
|
|
|
if !ok {
|
|
|
|
t.Fatal("GlobalPinInfo should have this cluster")
|
|
|
|
}
|
2017-02-08 17:04:08 +00:00
|
|
|
if inf.Status != api.TrackerStatusPinned {
|
2017-01-26 18:59:31 +00:00
|
|
|
t.Error("the GlobalPinInfo should show Pinned in all peers")
|
2016-12-21 13:30:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-01-30 12:12:25 +00:00
|
|
|
|
|
|
|
func TestClustersShutdown(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-01-30 12:12:25 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
err := c.Shutdown(ctx)
|
2017-01-30 12:12:25 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error("should be able to shutdown cleanly")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Shutdown 3 times
|
|
|
|
runF(t, clusters, f)
|
|
|
|
runF(t, clusters, f)
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
func TestClustersReplication(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-02-13 15:46:53 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
2018-01-12 17:04:46 +00:00
|
|
|
c.config.ReplicationFactorMin = nClusters - 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters - 1
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
// Why is replication factor nClusters - 1?
|
|
|
|
// Because that way we know that pinning nCluster
|
2017-03-27 13:07:12 +00:00
|
|
|
// pins with an strategy like numpins/disk
|
2017-02-13 15:46:53 +00:00
|
|
|
// will result in each peer holding locally exactly
|
|
|
|
// nCluster pins.
|
|
|
|
|
|
|
|
tmpCid, _ := cid.Decode(test.TestCid1)
|
|
|
|
prefix := tmpCid.Prefix()
|
|
|
|
|
|
|
|
for i := 0; i < nClusters; i++ {
|
|
|
|
// Pick a random cluster and hash
|
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer
|
|
|
|
h, err := prefix.Sum(randomBytes()) // create random cid
|
|
|
|
checkErr(t, err)
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[j].Pin(ctx, api.PinCid(h))
|
2017-02-13 15:46:53 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Error(err)
|
|
|
|
}
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
// check that it is held by exactly nClusters -1 peers
|
2018-06-27 04:03:15 +00:00
|
|
|
gpi, err := clusters[j].Status(ctx, h)
|
2017-02-13 15:46:53 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
numLocal := 0
|
|
|
|
numRemote := 0
|
|
|
|
for _, v := range gpi.PeerMap {
|
|
|
|
if v.Status == api.TrackerStatusPinned {
|
|
|
|
numLocal++
|
|
|
|
} else if v.Status == api.TrackerStatusRemote {
|
|
|
|
numRemote++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if numLocal != nClusters-1 {
|
|
|
|
t.Errorf("We wanted replication %d but it's only %d",
|
|
|
|
nClusters-1, numLocal)
|
|
|
|
}
|
|
|
|
|
|
|
|
if numRemote != 1 {
|
|
|
|
t.Errorf("We wanted 1 peer track as remote but %d do", numRemote)
|
|
|
|
}
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfos := c.tracker.StatusAll(ctx)
|
2017-02-13 15:46:53 +00:00
|
|
|
if len(pinfos) != nClusters {
|
|
|
|
t.Error("Pinfos does not have the expected pins")
|
|
|
|
}
|
|
|
|
numRemote := 0
|
|
|
|
numLocal := 0
|
|
|
|
for _, pi := range pinfos {
|
|
|
|
switch pi.Status {
|
|
|
|
case api.TrackerStatusPinned:
|
|
|
|
numLocal++
|
|
|
|
|
|
|
|
case api.TrackerStatusRemote:
|
|
|
|
numRemote++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if numLocal != nClusters-1 {
|
|
|
|
t.Errorf("Expected %d local pins but got %d", nClusters-1, numLocal)
|
2018-05-02 11:49:19 +00:00
|
|
|
t.Error(pinfos)
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if numRemote != 1 {
|
|
|
|
t.Errorf("Expected 1 remote pin but got %d", numRemote)
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
pins := c.Pins(ctx)
|
2017-02-13 15:46:53 +00:00
|
|
|
for _, pin := range pins {
|
|
|
|
allocs := pin.Allocations
|
|
|
|
if len(allocs) != nClusters-1 {
|
|
|
|
t.Errorf("Allocations are [%s]", allocs)
|
|
|
|
}
|
|
|
|
for _, a := range allocs {
|
|
|
|
if a == c.id {
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfo := c.tracker.Status(ctx, pin.Cid)
|
2017-02-13 15:46:53 +00:00
|
|
|
if pinfo.Status != api.TrackerStatusPinned {
|
|
|
|
t.Errorf("Peer %s was allocated but it is not pinning cid", c.id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
// This test checks that we pin with ReplicationFactorMax when
|
|
|
|
// we can
|
|
|
|
func TestClustersReplicationFactorMax(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-15 18:32:19 +00:00
|
|
|
if nClusters < 3 {
|
|
|
|
t.Skip("Need at least 3 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters - 1
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2018-01-15 18:32:19 +00:00
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
2018-06-27 04:03:15 +00:00
|
|
|
p, err := c.PinGet(ctx, h)
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(p.Allocations) != nClusters-1 {
|
|
|
|
t.Error("should have pinned nClusters - 1 allocations")
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.ReplicationFactorMin != 1 {
|
|
|
|
t.Error("rplMin should be 1")
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.ReplicationFactorMax != nClusters-1 {
|
2018-01-16 10:19:39 +00:00
|
|
|
t.Error("rplMax should be nClusters-1")
|
2018-01-15 18:32:19 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
2018-01-16 10:19:39 +00:00
|
|
|
// This tests checks that repinning something that is overpinned
|
|
|
|
// removes some allocations
|
|
|
|
func TestClustersReplicationFactorMaxLower(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-16 10:19:39 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay() // make sure we have places to pin
|
|
|
|
|
2018-01-16 10:19:39 +00:00
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-16 10:19:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2018-01-16 10:19:39 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
p1, err := clusters[0].PinGet(ctx, h)
|
2018-01-16 10:19:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(p1.Allocations) != nClusters {
|
|
|
|
t.Fatal("allocations should be nClusters")
|
|
|
|
}
|
|
|
|
|
2018-07-23 13:56:46 +00:00
|
|
|
pin := api.PinCid(h)
|
|
|
|
pin.ReplicationFactorMin = 1
|
|
|
|
pin.ReplicationFactorMax = 2
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[0].Pin(ctx, pin)
|
2018-01-16 10:19:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2018-01-16 10:19:39 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
p2, err := clusters[0].PinGet(ctx, h)
|
2018-01-16 10:19:39 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(p2.Allocations) != 2 {
|
|
|
|
t.Fatal("allocations should have been reduced to 2")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
// This test checks that when not all nodes are available,
|
|
|
|
// we pin in as many as we can aiming for ReplicationFactorMax
|
|
|
|
func TestClustersReplicationFactorInBetween(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-15 18:32:19 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
// Shutdown two peers
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[nClusters-1].Shutdown(ctx)
|
|
|
|
clusters[nClusters-2].Shutdown(ctx)
|
2018-01-15 18:32:19 +00:00
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-01-15 18:32:19 +00:00
|
|
|
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2018-01-15 18:32:19 +00:00
|
|
|
|
|
|
|
f := func(t *testing.T, c *Cluster) {
|
|
|
|
if c == clusters[nClusters-1] || c == clusters[nClusters-2] {
|
|
|
|
return
|
|
|
|
}
|
2018-06-27 04:03:15 +00:00
|
|
|
p, err := c.PinGet(ctx, h)
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(p.Allocations) != nClusters-2 {
|
2018-01-16 10:19:39 +00:00
|
|
|
t.Error("should have pinned nClusters-2 allocations")
|
2018-01-15 18:32:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if p.ReplicationFactorMin != 1 {
|
|
|
|
t.Error("rplMin should be 1")
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.ReplicationFactorMax != nClusters {
|
|
|
|
t.Error("rplMax should be nClusters")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
runF(t, clusters, f)
|
|
|
|
}
|
|
|
|
|
|
|
|
// This test checks that we do not pin something for which
|
|
|
|
// we cannot reach ReplicationFactorMin
|
|
|
|
func TestClustersReplicationFactorMin(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-15 18:32:19 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = nClusters - 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters
|
|
|
|
}
|
|
|
|
|
|
|
|
// Shutdown two peers
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[nClusters-1].Shutdown(ctx)
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[nClusters-2].Shutdown(ctx)
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-01-15 18:32:19 +00:00
|
|
|
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Error("Pin should have failed as rplMin cannot be satisfied")
|
|
|
|
}
|
|
|
|
t.Log(err)
|
|
|
|
if !strings.Contains(err.Error(), fmt.Sprintf("not enough peers to allocate CID")) {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// This tests checks that repinning something that has becomed
|
|
|
|
// underpinned actually changes nothing if it's sufficiently pinned
|
|
|
|
func TestClustersReplicationMinMaxNoRealloc(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-15 18:32:19 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
// Shutdown two peers
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[nClusters-1].Shutdown(ctx)
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[nClusters-2].Shutdown(ctx)
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-01-15 18:32:19 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
p, err := clusters[0].PinGet(ctx, h)
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(p.Allocations) != nClusters {
|
|
|
|
t.Error("allocations should still be nCluster even if not all available")
|
|
|
|
}
|
|
|
|
|
|
|
|
if p.ReplicationFactorMax != nClusters {
|
|
|
|
t.Error("rplMax should have not changed")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-16 10:19:39 +00:00
|
|
|
// This test checks that repinning something that has becomed
|
2018-01-15 18:32:19 +00:00
|
|
|
// underpinned does re-allocations when it's not sufficiently
|
|
|
|
// pinned anymore
|
|
|
|
func TestClustersReplicationMinMaxRealloc(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-15 18:32:19 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = 3
|
|
|
|
c.config.ReplicationFactorMax = 4
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay() // make sure metrics are in
|
|
|
|
|
2018-01-15 18:32:19 +00:00
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[0].Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2018-01-15 18:32:19 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
p, err := clusters[0].PinGet(ctx, h)
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
firstAllocations := p.Allocations
|
|
|
|
|
|
|
|
peerIDMap := make(map[peer.ID]*Cluster)
|
|
|
|
for _, a := range clusters {
|
|
|
|
peerIDMap[a.id] = a
|
|
|
|
}
|
|
|
|
|
|
|
|
// kill two of the allocations
|
|
|
|
alloc1 := peerIDMap[firstAllocations[0]]
|
|
|
|
alloc2 := peerIDMap[firstAllocations[1]]
|
|
|
|
safePeer := peerIDMap[firstAllocations[2]]
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
alloc1.Shutdown(ctx)
|
|
|
|
alloc2.Shutdown(ctx)
|
2018-01-15 18:32:19 +00:00
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-01-15 18:32:19 +00:00
|
|
|
|
|
|
|
// Repin - (although this might have been taken of if there was an alert
|
2018-06-27 04:03:15 +00:00
|
|
|
err = safePeer.Pin(ctx, api.PinCid(h))
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
p, err = safePeer.PinGet(ctx, h)
|
2018-01-15 18:32:19 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
secondAllocations := p.Allocations
|
|
|
|
|
|
|
|
strings1 := api.PeersToStrings(firstAllocations)
|
|
|
|
strings2 := api.PeersToStrings(secondAllocations)
|
|
|
|
sort.Strings(strings1)
|
|
|
|
sort.Strings(strings2)
|
|
|
|
t.Logf("Allocs1: %s", strings1)
|
|
|
|
t.Logf("Allocs2: %s", strings2)
|
|
|
|
|
|
|
|
if fmt.Sprintf("%s", strings1) == fmt.Sprintf("%s", strings2) {
|
|
|
|
t.Error("allocations should have changed")
|
|
|
|
}
|
|
|
|
|
2018-01-16 10:19:39 +00:00
|
|
|
lenSA := len(secondAllocations)
|
|
|
|
expected := minInt(nClusters-2, 4)
|
|
|
|
if lenSA != expected {
|
2018-03-16 16:37:39 +00:00
|
|
|
t.Errorf("Insufficient reallocation, could have allocated to %d peers but instead only allocated to %d peers", expected, lenSA)
|
2018-01-16 10:19:39 +00:00
|
|
|
}
|
|
|
|
|
2018-01-19 12:04:06 +00:00
|
|
|
if lenSA < 3 {
|
2018-01-15 18:32:19 +00:00
|
|
|
t.Error("allocations should be more than rplMin")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
// In this test we check that repinning something
|
|
|
|
// when a node has gone down will re-assign the pin
|
|
|
|
func TestClustersReplicationRealloc(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-02-13 15:46:53 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
2018-01-12 17:04:46 +00:00
|
|
|
c.config.ReplicationFactorMin = nClusters - 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters - 1
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
ttlDelay()
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
j := rand.Intn(nClusters)
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[j].Pin(ctx, api.PinCid(h))
|
2017-02-13 15:46:53 +00:00
|
|
|
if err != nil {
|
2017-03-09 13:44:14 +00:00
|
|
|
t.Fatal(err)
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Let the pin arrive
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
pin := clusters[j].Pins(ctx)[0]
|
2017-03-09 13:44:14 +00:00
|
|
|
pinSerial := pin.ToSerial()
|
|
|
|
allocs := sort.StringSlice(pinSerial.Allocations)
|
|
|
|
allocs.Sort()
|
|
|
|
allocsStr := fmt.Sprintf("%s", allocs)
|
|
|
|
|
|
|
|
// Re-pin should work and be allocated to the same
|
|
|
|
// nodes
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[j].Pin(ctx, api.PinCid(h))
|
2017-03-09 13:44:14 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
2017-03-09 13:44:14 +00:00
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-03-09 13:44:14 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
pin2 := clusters[j].Pins(ctx)[0]
|
2017-03-09 13:44:14 +00:00
|
|
|
pinSerial2 := pin2.ToSerial()
|
|
|
|
allocs2 := sort.StringSlice(pinSerial2.Allocations)
|
|
|
|
allocs2.Sort()
|
|
|
|
allocsStr2 := fmt.Sprintf("%s", allocs2)
|
|
|
|
if allocsStr != allocsStr2 {
|
|
|
|
t.Fatal("allocations changed without reason")
|
|
|
|
}
|
|
|
|
//t.Log(allocsStr)
|
|
|
|
//t.Log(allocsStr2)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
|
|
|
var killedClusterIndex int
|
|
|
|
// find someone that pinned it and kill that cluster
|
|
|
|
for i, c := range clusters {
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfo := c.tracker.Status(ctx, h)
|
2017-02-13 15:46:53 +00:00
|
|
|
if pinfo.Status == api.TrackerStatusPinned {
|
2017-03-09 13:44:14 +00:00
|
|
|
//t.Logf("Killing %s", c.id.Pretty())
|
2017-02-13 15:46:53 +00:00
|
|
|
killedClusterIndex = i
|
2018-06-27 04:03:15 +00:00
|
|
|
t.Logf("Shutting down %s", c.ID(ctx).ID)
|
|
|
|
c.Shutdown(ctx)
|
2017-03-09 13:44:14 +00:00
|
|
|
break
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-21 08:34:20 +00:00
|
|
|
// let metrics expire and give time for the cluster to
|
|
|
|
// see if they have lost the leader
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2017-03-09 13:44:14 +00:00
|
|
|
|
|
|
|
// Make sure we haven't killed our randomly
|
|
|
|
// selected cluster
|
|
|
|
for j == killedClusterIndex {
|
|
|
|
j = rand.Intn(nClusters)
|
|
|
|
}
|
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
// now pin should succeed
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[j].Pin(ctx, api.PinCid(h))
|
2017-02-13 15:46:53 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-03-09 13:44:14 +00:00
|
|
|
|
2017-02-13 15:46:53 +00:00
|
|
|
numPinned := 0
|
|
|
|
for i, c := range clusters {
|
|
|
|
if i == killedClusterIndex {
|
|
|
|
continue
|
|
|
|
}
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfo := c.tracker.Status(ctx, h)
|
2017-02-13 15:46:53 +00:00
|
|
|
if pinfo.Status == api.TrackerStatusPinned {
|
2017-03-09 13:44:14 +00:00
|
|
|
//t.Log(pinfo.Peer.Pretty())
|
2017-02-13 15:46:53 +00:00
|
|
|
numPinned++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if numPinned != nClusters-1 {
|
|
|
|
t.Error("pin should have been correctly re-assigned")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// In this test we try to pin something when there are not
|
|
|
|
// as many available peers a we need. It's like before, except
|
|
|
|
// more peers are killed.
|
|
|
|
func TestClustersReplicationNotEnoughPeers(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-02-13 15:46:53 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
2018-01-12 17:04:46 +00:00
|
|
|
c.config.ReplicationFactorMin = nClusters - 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters - 1
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
j := rand.Intn(nClusters)
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[j].Pin(ctx, api.PinCid(h))
|
2017-02-13 15:46:53 +00:00
|
|
|
if err != nil {
|
2017-02-28 15:01:26 +00:00
|
|
|
t.Fatal(err)
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Let the pin arrive
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Shutdown(ctx)
|
|
|
|
clusters[1].Shutdown(ctx)
|
2017-02-13 15:46:53 +00:00
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2017-02-28 15:01:26 +00:00
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
err = clusters[2].Pin(ctx, api.PinCid(h))
|
2017-02-13 15:46:53 +00:00
|
|
|
if err == nil {
|
|
|
|
t.Fatal("expected an error")
|
|
|
|
}
|
2018-01-12 17:04:46 +00:00
|
|
|
if !strings.Contains(err.Error(), "not enough peers to allocate") {
|
2017-02-13 15:46:53 +00:00
|
|
|
t.Error("different error than expected")
|
|
|
|
t.Error(err)
|
|
|
|
}
|
2017-03-13 14:55:52 +00:00
|
|
|
//t.Log(err)
|
2017-02-13 15:46:53 +00:00
|
|
|
}
|
2017-02-28 15:01:26 +00:00
|
|
|
|
|
|
|
func TestClustersRebalanceOnPeerDown(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2017-02-28 15:01:26 +00:00
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
2018-01-12 17:04:46 +00:00
|
|
|
c.config.ReplicationFactorMin = nClusters - 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters - 1
|
2017-02-28 15:01:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// pin something
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[0].Pin(ctx, api.PinCid(h))
|
2018-03-16 16:37:39 +00:00
|
|
|
pinDelay()
|
2017-02-28 15:01:26 +00:00
|
|
|
pinLocal := 0
|
|
|
|
pinRemote := 0
|
|
|
|
var localPinner peer.ID
|
|
|
|
var remotePinner peer.ID
|
|
|
|
var remotePinnerCluster *Cluster
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
status, _ := clusters[0].Status(ctx, h)
|
2017-02-28 15:01:26 +00:00
|
|
|
|
|
|
|
// check it was correctly pinned
|
|
|
|
for p, pinfo := range status.PeerMap {
|
|
|
|
if pinfo.Status == api.TrackerStatusPinned {
|
|
|
|
pinLocal++
|
|
|
|
localPinner = p
|
|
|
|
} else if pinfo.Status == api.TrackerStatusRemote {
|
|
|
|
pinRemote++
|
|
|
|
remotePinner = p
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if pinLocal != nClusters-1 || pinRemote != 1 {
|
|
|
|
t.Fatal("Not pinned as expected")
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
// kill the local pinner
|
2017-02-28 15:01:26 +00:00
|
|
|
for _, c := range clusters {
|
|
|
|
if c.id == localPinner {
|
2018-06-27 04:03:15 +00:00
|
|
|
c.Shutdown(ctx)
|
2017-02-28 15:01:26 +00:00
|
|
|
} else if c.id == remotePinner {
|
|
|
|
remotePinnerCluster = c
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-16 16:37:39 +00:00
|
|
|
delay()
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters) // in case we killed the leader
|
2017-02-28 15:01:26 +00:00
|
|
|
|
|
|
|
// It should be now pinned in the remote pinner
|
2018-06-27 04:03:15 +00:00
|
|
|
if s := remotePinnerCluster.tracker.Status(ctx, h).Status; s != api.TrackerStatusPinned {
|
2017-02-28 15:01:26 +00:00
|
|
|
t.Errorf("it should be pinned and is %s", s)
|
|
|
|
}
|
|
|
|
}
|
2018-01-18 02:49:35 +00:00
|
|
|
|
|
|
|
// Helper function for verifying cluster graph. Will only pass if exactly the
|
|
|
|
// peers in clusterIDs are fully connected to each other and the expected ipfs
|
|
|
|
// mock connectivity exists. Cluster peers not in clusterIDs are assumed to
|
|
|
|
// be disconnected and the graph should reflect this
|
|
|
|
func validateClusterGraph(t *testing.T, graph api.ConnectGraph, clusterIDs map[peer.ID]struct{}) {
|
|
|
|
// Check that all cluster peers see each other as peers
|
|
|
|
for id1, peers := range graph.ClusterLinks {
|
|
|
|
if _, ok := clusterIDs[id1]; !ok {
|
|
|
|
if len(peers) != 0 {
|
|
|
|
t.Errorf("disconnected peer %s is still connected in graph", id1)
|
|
|
|
}
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
fmt.Printf("id: %s, peers: %v\n", id1, peers)
|
|
|
|
if len(peers) > len(clusterIDs)-1 {
|
|
|
|
t.Errorf("More peers recorded in graph than expected")
|
|
|
|
}
|
|
|
|
// Make lookup index for peers connected to id1
|
|
|
|
peerIndex := make(map[peer.ID]struct{})
|
|
|
|
for _, peer := range peers {
|
|
|
|
peerIndex[peer] = struct{}{}
|
|
|
|
}
|
|
|
|
for id2 := range clusterIDs {
|
|
|
|
if _, ok := peerIndex[id2]; id1 != id2 && !ok {
|
|
|
|
t.Errorf("Expected graph to see peer %s connected to peer %s", id1, id2)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(graph.ClusterLinks) != nClusters {
|
|
|
|
t.Errorf("Unexpected number of cluster nodes in graph")
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that all cluster peers are recorded as nodes in the graph
|
|
|
|
for id := range clusterIDs {
|
|
|
|
if _, ok := graph.ClusterLinks[id]; !ok {
|
|
|
|
t.Errorf("Expected graph to record peer %s as a node", id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that the mocked ipfs swarm is recorded
|
|
|
|
if len(graph.IPFSLinks) != 1 {
|
|
|
|
t.Error("Expected exactly one ipfs peer for all cluster nodes, the mocked peer")
|
|
|
|
}
|
|
|
|
links, ok := graph.IPFSLinks[test.TestPeerID1]
|
|
|
|
if !ok {
|
|
|
|
t.Error("Expected the mocked ipfs peer to be a node in the graph")
|
|
|
|
} else {
|
|
|
|
if len(links) != 2 || links[0] != test.TestPeerID4 ||
|
|
|
|
links[1] != test.TestPeerID5 {
|
|
|
|
t.Error("Swarm peers of mocked ipfs are not those expected")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check that the cluster to ipfs connections are all recorded
|
|
|
|
for id := range clusterIDs {
|
|
|
|
if ipfsID, ok := graph.ClustertoIPFS[id]; !ok {
|
|
|
|
t.Errorf("Expected graph to record peer %s's ipfs connection", id)
|
|
|
|
} else {
|
|
|
|
if ipfsID != test.TestPeerID1 {
|
|
|
|
t.Errorf("Unexpected error %s", ipfsID)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if len(graph.ClustertoIPFS) > len(clusterIDs) {
|
|
|
|
t.Error("More cluster to ipfs links recorded in graph than expected")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// In this test we get a cluster graph report from a random peer in a healthy
|
|
|
|
// fully connected cluster and verify that it is formed as expected.
|
|
|
|
func TestClustersGraphConnected(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-18 02:49:35 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
|
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer to query
|
|
|
|
graph, err := clusters[j].ConnectGraph()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
clusterIDs := make(map[peer.ID]struct{})
|
|
|
|
for _, c := range clusters {
|
2018-06-27 04:03:15 +00:00
|
|
|
id := c.ID(ctx).ID
|
2018-01-18 02:49:35 +00:00
|
|
|
clusterIDs[id] = struct{}{}
|
|
|
|
}
|
|
|
|
validateClusterGraph(t, graph, clusterIDs)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Similar to the previous test we get a cluster graph report from a peer.
|
|
|
|
// However now 2 peers have been shutdown and so we do not expect to see
|
|
|
|
// them in the graph
|
|
|
|
func TestClustersGraphUnhealthy(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-01-18 02:49:35 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
if nClusters < 5 {
|
|
|
|
t.Skip("Need at least 5 peers")
|
|
|
|
}
|
|
|
|
|
|
|
|
j := rand.Intn(nClusters) // choose a random cluster peer to query
|
|
|
|
// chose the clusters to shutdown
|
|
|
|
discon1 := -1
|
|
|
|
discon2 := -1
|
|
|
|
for i := range clusters {
|
|
|
|
if i != j {
|
|
|
|
if discon1 == -1 {
|
|
|
|
discon1 = i
|
|
|
|
} else {
|
|
|
|
discon2 = i
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
clusters[discon1].Shutdown(ctx)
|
|
|
|
clusters[discon2].Shutdown(ctx)
|
2018-03-16 16:37:39 +00:00
|
|
|
|
2018-03-29 20:31:11 +00:00
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
2018-01-18 02:49:35 +00:00
|
|
|
|
|
|
|
graph, err := clusters[j].ConnectGraph()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
clusterIDs := make(map[peer.ID]struct{})
|
|
|
|
for i, c := range clusters {
|
|
|
|
if i == discon1 || i == discon2 {
|
|
|
|
continue
|
|
|
|
}
|
2018-06-27 04:03:15 +00:00
|
|
|
id := c.ID(ctx).ID
|
2018-01-18 02:49:35 +00:00
|
|
|
clusterIDs[id] = struct{}{}
|
|
|
|
}
|
|
|
|
validateClusterGraph(t, graph, clusterIDs)
|
|
|
|
}
|
2018-04-23 20:01:44 +00:00
|
|
|
|
|
|
|
// Check that the pin is not re-assigned when a node
|
|
|
|
// that has disabled repinning goes down.
|
|
|
|
func TestClustersDisabledRepinning(t *testing.T) {
|
2018-06-27 04:03:15 +00:00
|
|
|
ctx := context.Background()
|
2018-04-23 20:01:44 +00:00
|
|
|
clusters, mock := createClusters(t)
|
|
|
|
defer shutdownClusters(t, clusters, mock)
|
|
|
|
for _, c := range clusters {
|
|
|
|
c.config.ReplicationFactorMin = nClusters - 1
|
|
|
|
c.config.ReplicationFactorMax = nClusters - 1
|
|
|
|
c.config.DisableRepinning = true
|
|
|
|
}
|
|
|
|
|
|
|
|
ttlDelay()
|
|
|
|
|
|
|
|
j := rand.Intn(nClusters)
|
|
|
|
h, _ := cid.Decode(test.TestCid1)
|
2018-06-27 04:03:15 +00:00
|
|
|
err := clusters[j].Pin(ctx, api.PinCid(h))
|
2018-04-23 20:01:44 +00:00
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Let the pin arrive
|
|
|
|
pinDelay()
|
|
|
|
|
|
|
|
var killedClusterIndex int
|
|
|
|
// find someone that pinned it and kill that cluster
|
|
|
|
for i, c := range clusters {
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfo := c.tracker.Status(ctx, h)
|
2018-04-23 20:01:44 +00:00
|
|
|
if pinfo.Status == api.TrackerStatusPinned {
|
|
|
|
killedClusterIndex = i
|
2018-06-27 04:03:15 +00:00
|
|
|
t.Logf("Shutting down %s", c.ID(ctx).ID)
|
|
|
|
c.Shutdown(ctx)
|
2018-04-23 20:01:44 +00:00
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// let metrics expire and give time for the cluster to
|
|
|
|
// see if they have lost the leader
|
|
|
|
waitForLeaderAndMetrics(t, clusters)
|
|
|
|
|
|
|
|
// Make sure we haven't killed our randomly
|
|
|
|
// selected cluster
|
|
|
|
for j == killedClusterIndex {
|
|
|
|
j = rand.Intn(nClusters)
|
|
|
|
}
|
|
|
|
|
|
|
|
numPinned := 0
|
|
|
|
for i, c := range clusters {
|
|
|
|
if i == killedClusterIndex {
|
|
|
|
continue
|
|
|
|
}
|
2018-06-27 04:03:15 +00:00
|
|
|
pinfo := c.tracker.Status(ctx, h)
|
2018-04-23 20:01:44 +00:00
|
|
|
if pinfo.Status == api.TrackerStatusPinned {
|
|
|
|
//t.Log(pinfo.Peer.Pretty())
|
|
|
|
numPinned++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if numPinned != nClusters-2 {
|
|
|
|
t.Errorf("expected %d replicas for pin, got %d", nClusters-2, numPinned)
|
|
|
|
}
|
|
|
|
}
|