Feat pubsubmon: Extract MetricsWindow to utils module
License: MIT Signed-off-by: Hector Sanjuan <code@hector.link>
This commit is contained in:
parent
029cd77c27
commit
1886782530
|
@ -5,7 +5,6 @@ package basic
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
@ -14,6 +13,7 @@ import (
|
||||||
peer "github.com/libp2p/go-libp2p-peer"
|
peer "github.com/libp2p/go-libp2p-peer"
|
||||||
|
|
||||||
"github.com/ipfs/ipfs-cluster/api"
|
"github.com/ipfs/ipfs-cluster/api"
|
||||||
|
"github.com/ipfs/ipfs-cluster/monitor/util"
|
||||||
)
|
)
|
||||||
|
|
||||||
var logger = logging.Logger("monitor")
|
var logger = logging.Logger("monitor")
|
||||||
|
@ -21,65 +21,10 @@ var logger = logging.Logger("monitor")
|
||||||
// AlertChannelCap specifies how much buffer the alerts channel has.
|
// AlertChannelCap specifies how much buffer the alerts channel has.
|
||||||
var AlertChannelCap = 256
|
var AlertChannelCap = 256
|
||||||
|
|
||||||
// WindowCap specifies how many metrics to keep for given host and metric type
|
// DefaultWindowCap sets the amount of metrics to store per peer.
|
||||||
var WindowCap = 100
|
var DefaultWindowCap = 25
|
||||||
|
|
||||||
// peerMetrics is just a circular queue
|
type metricsByPeer map[peer.ID]*util.MetricsWindow
|
||||||
type peerMetrics struct {
|
|
||||||
last int
|
|
||||||
window []api.Metric
|
|
||||||
// mux sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func newPeerMetrics(windowCap int) *peerMetrics {
|
|
||||||
w := make([]api.Metric, 0, windowCap)
|
|
||||||
return &peerMetrics{0, w}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (pmets *peerMetrics) add(m api.Metric) {
|
|
||||||
// pmets.mux.Lock()
|
|
||||||
// defer pmets.mux.Unlock()
|
|
||||||
if len(pmets.window) < cap(pmets.window) {
|
|
||||||
pmets.window = append(pmets.window, m)
|
|
||||||
pmets.last = len(pmets.window) - 1
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// len == cap
|
|
||||||
pmets.last = (pmets.last + 1) % cap(pmets.window)
|
|
||||||
pmets.window[pmets.last] = m
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (pmets *peerMetrics) latest() (api.Metric, error) {
|
|
||||||
// pmets.mux.RLock()
|
|
||||||
// defer pmets.mux.RUnlock()
|
|
||||||
if len(pmets.window) == 0 {
|
|
||||||
logger.Warning("no metrics")
|
|
||||||
return api.Metric{}, errors.New("no metrics")
|
|
||||||
}
|
|
||||||
return pmets.window[pmets.last], nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ordered from newest to oldest
|
|
||||||
func (pmets *peerMetrics) all() []api.Metric {
|
|
||||||
// pmets.mux.RLock()
|
|
||||||
// pmets.mux.RUnlock()
|
|
||||||
wlen := len(pmets.window)
|
|
||||||
res := make([]api.Metric, 0, wlen)
|
|
||||||
if wlen == 0 {
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
for i := pmets.last; i >= 0; i-- {
|
|
||||||
res = append(res, pmets.window[i])
|
|
||||||
}
|
|
||||||
for i := wlen; i > pmets.last; i-- {
|
|
||||||
res = append(res, pmets.window[i])
|
|
||||||
}
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
type metricsByPeer map[peer.ID]*peerMetrics
|
|
||||||
|
|
||||||
// Monitor is a component in charge of monitoring peers, logging
|
// Monitor is a component in charge of monitoring peers, logging
|
||||||
// metrics and detecting failures
|
// metrics and detecting failures
|
||||||
|
@ -112,7 +57,7 @@ func NewMonitor(cfg *Config) (*Monitor, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if WindowCap <= 0 {
|
if DefaultWindowCap <= 0 {
|
||||||
panic("windowCap too small")
|
panic("windowCap too small")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -124,7 +69,7 @@ func NewMonitor(cfg *Config) (*Monitor, error) {
|
||||||
rpcReady: make(chan struct{}, 1),
|
rpcReady: make(chan struct{}, 1),
|
||||||
|
|
||||||
metrics: make(map[string]metricsByPeer),
|
metrics: make(map[string]metricsByPeer),
|
||||||
windowCap: WindowCap,
|
windowCap: DefaultWindowCap,
|
||||||
alerts: make(chan api.Alert, AlertChannelCap),
|
alerts: make(chan api.Alert, AlertChannelCap),
|
||||||
|
|
||||||
config: cfg,
|
config: cfg,
|
||||||
|
@ -178,14 +123,16 @@ func (mon *Monitor) LogMetric(m api.Metric) {
|
||||||
mbyp = make(metricsByPeer)
|
mbyp = make(metricsByPeer)
|
||||||
mon.metrics[name] = mbyp
|
mon.metrics[name] = mbyp
|
||||||
}
|
}
|
||||||
pmets, ok := mbyp[peer]
|
window, ok := mbyp[peer]
|
||||||
if !ok {
|
if !ok {
|
||||||
pmets = newPeerMetrics(mon.windowCap)
|
// We always lock the outer map, so we can use unsafe
|
||||||
mbyp[peer] = pmets
|
// MetricsWindow.
|
||||||
|
window = util.NewMetricsWindow(mon.windowCap, false)
|
||||||
|
mbyp[peer] = window
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debugf("logged '%s' metric from '%s'. Expires on %d", name, peer, m.Expire)
|
logger.Debugf("logged '%s' metric from '%s'. Expires on %d", name, peer, m.Expire)
|
||||||
pmets.add(m)
|
window.Add(m)
|
||||||
}
|
}
|
||||||
|
|
||||||
// func (mon *Monitor) getLastMetric(name string, p peer.ID) api.Metric {
|
// func (mon *Monitor) getLastMetric(name string, p peer.ID) api.Metric {
|
||||||
|
@ -203,11 +150,11 @@ func (mon *Monitor) LogMetric(m api.Metric) {
|
||||||
// return emptyMetric
|
// return emptyMetric
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// pmets, ok := mbyp[p]
|
// window, ok := mbyp[p]
|
||||||
// if !ok {
|
// if !ok {
|
||||||
// return emptyMetric
|
// return emptyMetric
|
||||||
// }
|
// }
|
||||||
// metric, err := pmets.latest()
|
// metric, err := window.Latest()
|
||||||
// if err != nil {
|
// if err != nil {
|
||||||
// return emptyMetric
|
// return emptyMetric
|
||||||
// }
|
// }
|
||||||
|
@ -242,11 +189,11 @@ func (mon *Monitor) LastMetrics(name string) []api.Metric {
|
||||||
|
|
||||||
// only show metrics for current set of peers
|
// only show metrics for current set of peers
|
||||||
for _, peer := range peers {
|
for _, peer := range peers {
|
||||||
peerMetrics, ok := mbyp[peer]
|
window, ok := mbyp[peer]
|
||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
last, err := peerMetrics.latest()
|
last, err := window.Latest()
|
||||||
if err != nil || last.Discard() {
|
if err != nil || last.Discard() {
|
||||||
logger.Warningf("no valid last metric for peer: %+v", last)
|
logger.Warningf("no valid last metric for peer: %+v", last)
|
||||||
continue
|
continue
|
||||||
|
@ -308,11 +255,11 @@ func (mon *Monitor) checkMetrics(peers []peer.ID, metricName string) {
|
||||||
// for each of the given current peers
|
// for each of the given current peers
|
||||||
for _, p := range peers {
|
for _, p := range peers {
|
||||||
// get metrics for that peer
|
// get metrics for that peer
|
||||||
pMetrics, ok := metricsByPeer[p]
|
window, ok := metricsByPeer[p]
|
||||||
if !ok { // no metrics from this peer
|
if !ok { // no metrics from this peer
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
last, err := pMetrics.latest()
|
last, err := window.Latest()
|
||||||
if err != nil { // no metrics for this peer
|
if err != nil { // no metrics for this peer
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
87
monitor/util/metrics_window.go
Normal file
87
monitor/util/metrics_window.go
Normal file
|
@ -0,0 +1,87 @@
|
||||||
|
// Package util provides common functionality for monitoring components.
|
||||||
|
package util
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/ipfs/ipfs-cluster/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrNoMetrics = errors.New("no metrics have been added to this window")
|
||||||
|
|
||||||
|
// MetricsWindow implements a circular queue to store metrics.
|
||||||
|
type MetricsWindow struct {
|
||||||
|
last int
|
||||||
|
|
||||||
|
safe bool
|
||||||
|
windowLock sync.RWMutex
|
||||||
|
window []api.Metric
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMetricsWindow creates an instance with the given
|
||||||
|
// window capacity. The safe indicates whether we use a lock
|
||||||
|
// for concurrent operations.
|
||||||
|
func NewMetricsWindow(windowCap int, safe bool) *MetricsWindow {
|
||||||
|
w := make([]api.Metric, 0, windowCap)
|
||||||
|
return &MetricsWindow{
|
||||||
|
last: 0,
|
||||||
|
safe: safe,
|
||||||
|
window: w,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add adds a new metric to the window. If the window capacity
|
||||||
|
// has been reached, the oldest metric (by the time it was added),
|
||||||
|
// will be discarded.
|
||||||
|
func (mw *MetricsWindow) Add(m api.Metric) {
|
||||||
|
if mw.safe {
|
||||||
|
mw.windowLock.Lock()
|
||||||
|
defer mw.windowLock.Unlock()
|
||||||
|
}
|
||||||
|
if len(mw.window) < cap(mw.window) {
|
||||||
|
mw.window = append(mw.window, m)
|
||||||
|
mw.last = len(mw.window) - 1
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// len == cap
|
||||||
|
mw.last = (mw.last + 1) % cap(mw.window)
|
||||||
|
mw.window[mw.last] = m
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Latest returns the last metric added. It returns an error
|
||||||
|
// if no metrics were added.
|
||||||
|
func (mw *MetricsWindow) Latest() (api.Metric, error) {
|
||||||
|
if mw.safe {
|
||||||
|
mw.windowLock.Lock()
|
||||||
|
defer mw.windowLock.Unlock()
|
||||||
|
}
|
||||||
|
if len(mw.window) == 0 {
|
||||||
|
return api.Metric{}, ErrNoMetrics
|
||||||
|
}
|
||||||
|
return mw.window[mw.last], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// All returns all the metrics in the window, in the inverse order
|
||||||
|
// they were Added. That is, result[0] will be the last added
|
||||||
|
// metric.
|
||||||
|
func (mw *MetricsWindow) All() []api.Metric {
|
||||||
|
if mw.safe {
|
||||||
|
mw.windowLock.Lock()
|
||||||
|
defer mw.windowLock.Unlock()
|
||||||
|
}
|
||||||
|
wlen := len(mw.window)
|
||||||
|
res := make([]api.Metric, 0, wlen)
|
||||||
|
if wlen == 0 {
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
for i := mw.last; i >= 0; i-- {
|
||||||
|
res = append(res, mw.window[i])
|
||||||
|
}
|
||||||
|
for i := wlen - 1; i > mw.last; i-- {
|
||||||
|
res = append(res, mw.window[i])
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
71
monitor/util/metrics_window_test.go
Normal file
71
monitor/util/metrics_window_test.go
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
package util
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/ipfs/ipfs-cluster/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMetricsWindow(t *testing.T) {
|
||||||
|
mw := NewMetricsWindow(4)
|
||||||
|
|
||||||
|
_, err := mw.Latest()
|
||||||
|
if err != ErrNoMetrics {
|
||||||
|
t.Error("expected ErrNoMetrics")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(mw.All()) != 0 {
|
||||||
|
t.Error("expected 0 metrics")
|
||||||
|
}
|
||||||
|
|
||||||
|
metr := api.Metric{
|
||||||
|
Name: "test",
|
||||||
|
Peer: "peer1",
|
||||||
|
Value: "1",
|
||||||
|
Valid: true,
|
||||||
|
}
|
||||||
|
metr.SetTTL(5)
|
||||||
|
|
||||||
|
mw.Add(metr)
|
||||||
|
|
||||||
|
metr2, err := mw.Latest()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if metr2.Value != "1" {
|
||||||
|
t.Error("expected different value")
|
||||||
|
}
|
||||||
|
|
||||||
|
metr.Value = "2"
|
||||||
|
mw.Add(metr)
|
||||||
|
metr.Value = "3"
|
||||||
|
mw.Add(metr)
|
||||||
|
|
||||||
|
all := mw.All()
|
||||||
|
if len(all) != 3 {
|
||||||
|
t.Fatal("should only be storing 3 metrics")
|
||||||
|
}
|
||||||
|
|
||||||
|
if all[0].Value != "3" {
|
||||||
|
t.Error("newest metric should be first")
|
||||||
|
}
|
||||||
|
|
||||||
|
if all[1].Value != "2" {
|
||||||
|
t.Error("older metric should be second")
|
||||||
|
}
|
||||||
|
|
||||||
|
metr.Value = "4"
|
||||||
|
mw.Add(metr)
|
||||||
|
metr.Value = "5"
|
||||||
|
mw.Add(metr)
|
||||||
|
|
||||||
|
all = mw.All()
|
||||||
|
if len(all) != 4 {
|
||||||
|
t.Fatal("should only be storing 4 metrics")
|
||||||
|
}
|
||||||
|
|
||||||
|
if all[len(all)-1].Value != "2" {
|
||||||
|
t.Error("oldest metric should be 2")
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user