crdt: Implement proper Batch commit on shutdown
This implements committing batches on shutdown properly. Now the batchWorker will only finish when there are no more things queued to be included in the final batch(es). LogPin/Unpin operations will fail while we are shutting down and they cannot be included in the batch.
This commit is contained in:
parent
a393ebd8d8
commit
5e7a694cd1
|
@ -52,6 +52,7 @@ type batchItem struct {
|
|||
ctx context.Context
|
||||
isPin bool // pin or unpin
|
||||
pin api.Pin
|
||||
batched chan error // notify if item was sent for batching
|
||||
}
|
||||
|
||||
// Consensus implement ipfscluster.Consensus and provides the facility to add
|
||||
|
@ -85,7 +86,10 @@ type Consensus struct {
|
|||
rpcReady chan struct{}
|
||||
stateReady chan struct{}
|
||||
readyCh chan struct{}
|
||||
|
||||
sendToBatchCh chan batchItem
|
||||
batchItemCh chan batchItem
|
||||
batchingDone chan struct{}
|
||||
|
||||
shutdownLock sync.RWMutex
|
||||
shutdown bool
|
||||
|
@ -145,7 +149,9 @@ func New(
|
|||
rpcReady: make(chan struct{}, 1),
|
||||
readyCh: make(chan struct{}, 1),
|
||||
stateReady: make(chan struct{}, 1),
|
||||
sendToBatchCh: make(chan batchItem),
|
||||
batchItemCh: make(chan batchItem, cfg.Batching.MaxQueueSize),
|
||||
batchingDone: make(chan struct{}),
|
||||
}
|
||||
|
||||
go css.setup()
|
||||
|
@ -318,6 +324,7 @@ func (css *Consensus) setup() {
|
|||
css.config.Batching.MaxBatchSize,
|
||||
css.config.Batching.MaxBatchAge.String(),
|
||||
)
|
||||
go css.sendToBatchWorker()
|
||||
go css.batchWorker()
|
||||
}
|
||||
|
||||
|
@ -340,13 +347,10 @@ func (css *Consensus) Shutdown(ctx context.Context) error {
|
|||
|
||||
logger.Info("stopping Consensus component")
|
||||
|
||||
// Cancel
|
||||
// Cancel the batching code
|
||||
css.batchingCancel()
|
||||
if css.config.batchingEnabled() {
|
||||
logger.Info("committing pending batches")
|
||||
if err := css.batchingState.Commit(css.ctx); err != nil {
|
||||
logger.Errorf("error committing batch before shutdown: %w", err)
|
||||
}
|
||||
<-css.batchingDone
|
||||
}
|
||||
|
||||
css.cancel()
|
||||
|
@ -430,16 +434,14 @@ func (css *Consensus) LogPin(ctx context.Context, pin api.Pin) error {
|
|||
defer span.End()
|
||||
|
||||
if css.config.batchingEnabled() {
|
||||
select {
|
||||
case css.batchItemCh <- batchItem{
|
||||
batched := make(chan error)
|
||||
css.sendToBatchCh <- batchItem{
|
||||
ctx: ctx,
|
||||
isPin: true,
|
||||
pin: pin,
|
||||
}:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("error pinning: %w", ErrMaxQueueSizeReached)
|
||||
batched: batched,
|
||||
}
|
||||
return <-batched
|
||||
}
|
||||
|
||||
return css.state.Add(ctx, pin)
|
||||
|
@ -451,23 +453,50 @@ func (css *Consensus) LogUnpin(ctx context.Context, pin api.Pin) error {
|
|||
defer span.End()
|
||||
|
||||
if css.config.batchingEnabled() {
|
||||
select {
|
||||
case css.batchItemCh <- batchItem{
|
||||
batched := make(chan error)
|
||||
css.sendToBatchCh <- batchItem{
|
||||
ctx: ctx,
|
||||
isPin: false,
|
||||
pin: pin,
|
||||
}:
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("error unpinning: %w", ErrMaxQueueSizeReached)
|
||||
batched: batched,
|
||||
}
|
||||
return <-batched
|
||||
}
|
||||
|
||||
return css.state.Rm(ctx, pin.Cid)
|
||||
}
|
||||
|
||||
func (css *Consensus) sendToBatchWorker() {
|
||||
for {
|
||||
select {
|
||||
case <-css.batchingCtx.Done():
|
||||
close(css.batchItemCh)
|
||||
// This will stay here forever to catch any pins sent
|
||||
// while shutting down.
|
||||
for bi := range css.sendToBatchCh {
|
||||
bi.batched <- errors.New("shutting down. Pin could not be batched")
|
||||
close(bi.batched)
|
||||
}
|
||||
|
||||
return
|
||||
case bi := <-css.sendToBatchCh:
|
||||
select {
|
||||
case css.batchItemCh <- bi:
|
||||
close(bi.batched) // no error
|
||||
default: // queue is full
|
||||
err := fmt.Errorf("error batching item: %w", ErrMaxQueueSizeReached)
|
||||
logger.Error(err)
|
||||
bi.batched <- err
|
||||
close(bi.batched)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Launched in setup as a goroutine.
|
||||
func (css *Consensus) batchWorker() {
|
||||
defer close(css.batchingDone)
|
||||
|
||||
maxSize := css.config.Batching.MaxBatchSize
|
||||
maxAge := css.config.Batching.MaxBatchAge
|
||||
batchCurSize := 0
|
||||
|
@ -478,12 +507,36 @@ func (css *Consensus) batchWorker() {
|
|||
<-batchTimer.C
|
||||
}
|
||||
|
||||
// Add/Rm from state
|
||||
addToBatch := func(bi batchItem) error {
|
||||
var err error
|
||||
if bi.isPin {
|
||||
err = css.batchingState.Add(bi.ctx, bi.pin)
|
||||
} else {
|
||||
err = css.batchingState.Rm(bi.ctx, bi.pin.Cid)
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("error batching: %s (%s, isPin: %s)", err, bi.pin.Cid, bi.isPin)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-css.batchingCtx.Done():
|
||||
if !batchTimer.Stop() {
|
||||
<-batchTimer.C
|
||||
// Drain batchItemCh for missing things to be batched
|
||||
for batchItem := range css.batchItemCh {
|
||||
err := addToBatch(batchItem)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
batchCurSize++
|
||||
}
|
||||
if err := css.batchingState.Commit(css.ctx); err != nil {
|
||||
logger.Errorf("error committing batch during shutdown: %s", err)
|
||||
}
|
||||
logger.Infof("batch commit (shutdown): %d items", batchCurSize)
|
||||
|
||||
return
|
||||
case batchItem := <-css.batchItemCh:
|
||||
// First item in batch. Start the timer
|
||||
|
@ -491,15 +544,8 @@ func (css *Consensus) batchWorker() {
|
|||
batchTimer.Reset(maxAge)
|
||||
}
|
||||
|
||||
// Add/Rm from state
|
||||
var err error
|
||||
if batchItem.isPin {
|
||||
err = css.batchingState.Add(batchItem.ctx, batchItem.pin)
|
||||
} else {
|
||||
err = css.batchingState.Rm(batchItem.ctx, batchItem.pin.Cid)
|
||||
}
|
||||
err := addToBatch(batchItem)
|
||||
if err != nil {
|
||||
logger.Errorf("error batching: %s (%s, isPin: %s)", err, batchItem.pin.Cid, batchItem.isPin)
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user