2018-05-08 09:38:12 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
2019-06-14 10:41:11 +00:00
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
|
|
"github.com/ipfs/ipfs-cluster/test"
|
|
|
|
|
|
|
|
peer "github.com/libp2p/go-libp2p-core/peer"
|
2018-05-08 09:38:12 +00:00
|
|
|
)
|
|
|
|
|
2019-06-17 09:37:58 +00:00
|
|
|
func TestChecker_CheckPeers(t *testing.T) {
|
2019-06-10 11:30:34 +00:00
|
|
|
t.Run("check with single metric", func(t *testing.T) {
|
|
|
|
metrics := NewStore()
|
2022-01-28 15:38:29 +00:00
|
|
|
checker := NewChecker(context.Background(), metrics)
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
metr := &api.Metric{
|
|
|
|
Name: "ping",
|
|
|
|
Peer: test.PeerID1,
|
|
|
|
Value: "1",
|
|
|
|
Valid: true,
|
|
|
|
}
|
|
|
|
metr.SetTTL(2 * time.Second)
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
metrics.Add(metr)
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
checker.CheckPeers([]peer.ID{test.PeerID1})
|
|
|
|
select {
|
|
|
|
case <-checker.Alerts():
|
|
|
|
t.Error("there should not be an alert yet")
|
|
|
|
default:
|
|
|
|
}
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
err := checker.CheckPeers([]peer.ID{test.PeerID1})
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
select {
|
|
|
|
case <-checker.Alerts():
|
|
|
|
default:
|
|
|
|
t.Error("an alert should have been triggered")
|
|
|
|
}
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
checker.CheckPeers([]peer.ID{test.PeerID2})
|
|
|
|
select {
|
|
|
|
case <-checker.Alerts():
|
|
|
|
t.Error("there should not be alerts for different peer")
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
})
|
2018-05-08 09:38:12 +00:00
|
|
|
}
|
|
|
|
|
2019-06-17 09:37:58 +00:00
|
|
|
func TestChecker_CheckAll(t *testing.T) {
|
|
|
|
t.Run("checkall with single metric", func(t *testing.T) {
|
|
|
|
metrics := NewStore()
|
2022-01-28 15:38:29 +00:00
|
|
|
checker := NewChecker(context.Background(), metrics)
|
2019-06-17 09:37:58 +00:00
|
|
|
|
|
|
|
metr := &api.Metric{
|
|
|
|
Name: "ping",
|
|
|
|
Peer: test.PeerID1,
|
|
|
|
Value: "1",
|
|
|
|
Valid: true,
|
|
|
|
}
|
|
|
|
metr.SetTTL(2 * time.Second)
|
|
|
|
|
|
|
|
metrics.Add(metr)
|
|
|
|
|
|
|
|
checker.CheckAll()
|
|
|
|
select {
|
|
|
|
case <-checker.Alerts():
|
|
|
|
t.Error("there should not be an alert yet")
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
|
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
err := checker.CheckAll()
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-checker.Alerts():
|
|
|
|
default:
|
|
|
|
t.Error("an alert should have been triggered")
|
|
|
|
}
|
|
|
|
|
|
|
|
checker.CheckAll()
|
|
|
|
select {
|
|
|
|
case <-checker.Alerts():
|
|
|
|
t.Error("there should not be alerts for different peer")
|
|
|
|
default:
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-03-11 09:00:21 +00:00
|
|
|
func TestChecker_Watch(t *testing.T) {
|
2018-05-08 09:38:12 +00:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
metrics := NewStore()
|
2022-01-28 15:38:29 +00:00
|
|
|
checker := NewChecker(context.Background(), metrics)
|
2018-05-08 09:38:12 +00:00
|
|
|
|
2019-02-27 17:04:35 +00:00
|
|
|
metr := &api.Metric{
|
2019-06-17 09:37:58 +00:00
|
|
|
Name: "ping",
|
2019-02-27 20:19:10 +00:00
|
|
|
Peer: test.PeerID1,
|
2018-05-08 09:38:12 +00:00
|
|
|
Value: "1",
|
|
|
|
Valid: true,
|
|
|
|
}
|
|
|
|
metr.SetTTL(100 * time.Millisecond)
|
|
|
|
metrics.Add(metr)
|
|
|
|
|
2018-06-27 04:03:15 +00:00
|
|
|
peersF := func(context.Context) ([]peer.ID, error) {
|
2019-02-27 20:19:10 +00:00
|
|
|
return []peer.ID{test.PeerID1}, nil
|
2018-05-08 09:38:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
go checker.Watch(ctx, peersF, 200*time.Millisecond)
|
|
|
|
|
|
|
|
select {
|
|
|
|
case a := <-checker.Alerts():
|
|
|
|
t.Log("received alert:", a)
|
|
|
|
case <-ctx.Done():
|
|
|
|
t.Fatal("should have received an alert")
|
|
|
|
}
|
|
|
|
}
|
2019-03-11 09:00:21 +00:00
|
|
|
|
|
|
|
func TestChecker_Failed(t *testing.T) {
|
2019-03-21 10:42:56 +00:00
|
|
|
t.Run("standard failure check", func(t *testing.T) {
|
|
|
|
metrics := NewStore()
|
2022-01-28 15:38:29 +00:00
|
|
|
checker := NewChecker(context.Background(), metrics)
|
2019-03-11 09:00:21 +00:00
|
|
|
|
2022-01-28 15:38:29 +00:00
|
|
|
metrics.Add(makePeerMetric(test.PeerID1, "1", 100*time.Millisecond))
|
|
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
got := checker.FailedMetric("ping", test.PeerID1)
|
|
|
|
if got {
|
|
|
|
t.Error("should not have failed so soon")
|
2019-04-17 06:39:00 +00:00
|
|
|
}
|
2022-01-28 15:38:29 +00:00
|
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
got = checker.FailedMetric("ping", test.PeerID1)
|
|
|
|
if !got {
|
|
|
|
t.Error("should have failed")
|
2019-06-10 11:30:34 +00:00
|
|
|
}
|
|
|
|
})
|
2019-04-17 06:39:00 +00:00
|
|
|
}
|
|
|
|
|
2019-06-10 04:41:39 +00:00
|
|
|
func TestChecker_alert(t *testing.T) {
|
|
|
|
t.Run("remove peer from store after alert", func(t *testing.T) {
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
|
|
|
defer cancel()
|
|
|
|
|
|
|
|
metrics := NewStore()
|
2022-01-28 15:38:29 +00:00
|
|
|
checker := NewChecker(ctx, metrics)
|
2019-06-10 04:41:39 +00:00
|
|
|
|
|
|
|
metr := &api.Metric{
|
2019-06-17 09:37:58 +00:00
|
|
|
Name: "ping",
|
2019-06-10 04:41:39 +00:00
|
|
|
Peer: test.PeerID1,
|
|
|
|
Value: "1",
|
|
|
|
Valid: true,
|
|
|
|
}
|
|
|
|
metr.SetTTL(100 * time.Millisecond)
|
|
|
|
metrics.Add(metr)
|
|
|
|
|
|
|
|
peersF := func(context.Context) ([]peer.ID, error) {
|
|
|
|
return []peer.ID{test.PeerID1}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
go checker.Watch(ctx, peersF, 200*time.Millisecond)
|
|
|
|
|
|
|
|
var alertCount int
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case a := <-checker.Alerts():
|
|
|
|
t.Log("received alert:", a)
|
|
|
|
alertCount++
|
2019-06-10 11:54:48 +00:00
|
|
|
if alertCount > MaxAlertThreshold {
|
|
|
|
t.Fatalf("there should no more than %d alert", MaxAlertThreshold)
|
2019-06-10 04:41:39 +00:00
|
|
|
}
|
|
|
|
case <-ctx.Done():
|
|
|
|
if alertCount < 1 {
|
|
|
|
t.Fatal("should have received an alert")
|
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2019-06-10 11:30:34 +00:00
|
|
|
func makePeerMetric(pid peer.ID, value string, ttl time.Duration) *api.Metric {
|
2019-03-11 09:00:21 +00:00
|
|
|
metr := &api.Metric{
|
|
|
|
Name: "ping",
|
|
|
|
Peer: pid,
|
|
|
|
Value: value,
|
|
|
|
Valid: true,
|
|
|
|
}
|
2019-06-10 11:30:34 +00:00
|
|
|
metr.SetTTL(ttl)
|
2019-03-11 09:00:21 +00:00
|
|
|
return metr
|
|
|
|
}
|