d57b81490f
Since the beginning, we have used a Go map to store the shared state (pinset) in memory. The mapstate knew how to serialize itself so that libp2p-raft would know how to write to disk when it: * Saved snapshots of the state on shutdown * Sent the state to a newcomer peer hashicorp.Raft assumes an in-memory state which is snapshotted from time to time and read from disk on boot. This commit adds a `dsstate` implementation of the state interface using `go-datastore`. This allows to effortlessly switch to a disk-backed state in the future (as we will need), and also have at our disposal the different implementations and utilities of Datastore for fine-tuning (caching, batching etc.). `mapstate` has been reworked to use dsstate. Ideally, we would not even need `mapstate`, as it would suffice to initialize `dsstate` with a `MapDatastore`. BUT, we still need it separate to be able to auto-migrate to the new format. This will be the last migration with the current system. Once this has been released and users have been able to upgrade we will just remove `mapstate` as it is now. License: MIT Signed-off-by: Hector Sanjuan <code@hector.link>
240 lines
5.4 KiB
Go
240 lines
5.4 KiB
Go
package mapstate
|
|
|
|
// To add a new state format
|
|
// - implement the previous format's "next" function to the new format
|
|
// - implement the new format's unmarshal function
|
|
// - add a case to the switch statement for the previous format version
|
|
// - update the code copying the from mapStateVx to mapState
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
|
|
msgpack "github.com/multiformats/go-multicodec/msgpack"
|
|
|
|
"github.com/ipfs/ipfs-cluster/api"
|
|
)
|
|
|
|
// Instances of migrateable can be read from a serialized format and migrated
|
|
// to other state formats
|
|
type migrateable interface {
|
|
next() migrateable
|
|
unmarshal(io.Reader) error
|
|
}
|
|
|
|
/* V1 */
|
|
|
|
type mapStateV1 struct {
|
|
Version int
|
|
PinMap map[string]struct{}
|
|
}
|
|
|
|
// Unmarshal the serialization of a v1 state
|
|
func (st *mapStateV1) unmarshal(r io.Reader) error {
|
|
dec := msgpack.Multicodec(msgpack.DefaultMsgpackHandle()).Decoder(r)
|
|
return dec.Decode(st)
|
|
}
|
|
|
|
// Migrate from v1 to v2
|
|
func (st *mapStateV1) next() migrateable {
|
|
var mst2 mapStateV2
|
|
mst2.PinMap = make(map[string]pinSerialV2)
|
|
for k := range st.PinMap {
|
|
mst2.PinMap[k] = pinSerialV2{
|
|
Cid: k,
|
|
Allocations: []string{},
|
|
ReplicationFactor: -1,
|
|
}
|
|
}
|
|
return &mst2
|
|
}
|
|
|
|
/* V2 */
|
|
|
|
type pinSerialV2 struct {
|
|
Cid string `json:"cid"`
|
|
Name string `json:"name"`
|
|
Allocations []string `json:"allocations"`
|
|
ReplicationFactor int `json:"replication_factor"`
|
|
}
|
|
|
|
type mapStateV2 struct {
|
|
PinMap map[string]pinSerialV2
|
|
Version int
|
|
}
|
|
|
|
func (st *mapStateV2) unmarshal(r io.Reader) error {
|
|
dec := msgpack.Multicodec(msgpack.DefaultMsgpackHandle()).Decoder(r)
|
|
return dec.Decode(st)
|
|
}
|
|
|
|
func (st *mapStateV2) next() migrateable {
|
|
var mst3 mapStateV3
|
|
mst3.PinMap = make(map[string]pinSerialV3)
|
|
for k, v := range st.PinMap {
|
|
mst3.PinMap[k] = pinSerialV3{
|
|
Cid: v.Cid,
|
|
Name: v.Name,
|
|
Allocations: v.Allocations,
|
|
ReplicationFactorMin: v.ReplicationFactor,
|
|
ReplicationFactorMax: v.ReplicationFactor,
|
|
}
|
|
}
|
|
return &mst3
|
|
}
|
|
|
|
/* V3 */
|
|
|
|
type pinSerialV3 struct {
|
|
Cid string `json:"cid"`
|
|
Name string `json:"name"`
|
|
Allocations []string `json:"allocations"`
|
|
ReplicationFactorMin int `json:"replication_factor_min"`
|
|
ReplicationFactorMax int `json:"replication_factor_max"`
|
|
}
|
|
|
|
type mapStateV3 struct {
|
|
PinMap map[string]pinSerialV3
|
|
Version int
|
|
}
|
|
|
|
func (st *mapStateV3) unmarshal(r io.Reader) error {
|
|
dec := msgpack.Multicodec(msgpack.DefaultMsgpackHandle()).Decoder(r)
|
|
return dec.Decode(st)
|
|
}
|
|
|
|
func (st *mapStateV3) next() migrateable {
|
|
var mst4 mapStateV4
|
|
mst4.PinMap = make(map[string]pinSerialV4)
|
|
for k, v := range st.PinMap {
|
|
mst4.PinMap[k] = pinSerialV4{
|
|
Cid: v.Cid,
|
|
Name: v.Name,
|
|
Allocations: v.Allocations,
|
|
ReplicationFactorMin: v.ReplicationFactorMin,
|
|
ReplicationFactorMax: v.ReplicationFactorMax,
|
|
Recursive: true,
|
|
}
|
|
}
|
|
return &mst4
|
|
}
|
|
|
|
/* V4 */
|
|
|
|
type pinSerialV4 struct {
|
|
Cid string `json:"cid"`
|
|
Name string `json:"name"`
|
|
Allocations []string `json:"allocations"`
|
|
ReplicationFactorMin int `json:"replication_factor_min"`
|
|
ReplicationFactorMax int `json:"replication_factor_max"`
|
|
Recursive bool `json:"recursive"`
|
|
}
|
|
|
|
type mapStateV4 struct {
|
|
PinMap map[string]pinSerialV4
|
|
Version int
|
|
}
|
|
|
|
func (st *mapStateV4) unmarshal(r io.Reader) error {
|
|
dec := msgpack.Multicodec(msgpack.DefaultMsgpackHandle()).Decoder(r)
|
|
return dec.Decode(st)
|
|
}
|
|
|
|
func (st *mapStateV4) next() migrateable {
|
|
var mst5 mapStateV5
|
|
mst5.PinMap = make(map[string]api.PinSerial)
|
|
for k, v := range st.PinMap {
|
|
pinsv5 := api.PinSerial{}
|
|
pinsv5.Cid = v.Cid
|
|
pinsv5.Type = uint64(api.DataType)
|
|
pinsv5.Allocations = v.Allocations
|
|
|
|
// Encountered pins with Recursive=false
|
|
// in previous states. Since we do not support
|
|
// non recursive pins yet, we fix it by
|
|
// harcoding MaxDepth.
|
|
pinsv5.MaxDepth = -1
|
|
|
|
// Options
|
|
pinsv5.Name = v.Name
|
|
pinsv5.ReplicationFactorMin = v.ReplicationFactorMin
|
|
pinsv5.ReplicationFactorMax = v.ReplicationFactorMax
|
|
|
|
mst5.PinMap[k] = pinsv5
|
|
}
|
|
return &mst5
|
|
}
|
|
|
|
/* V5 */
|
|
|
|
type mapStateV5 struct {
|
|
PinMap map[string]api.PinSerial // this has not changed
|
|
Version int
|
|
}
|
|
|
|
func (st *mapStateV5) unmarshal(r io.Reader) error {
|
|
dec := msgpack.Multicodec(msgpack.DefaultMsgpackHandle()).Decoder(r)
|
|
return dec.Decode(st)
|
|
}
|
|
|
|
func (st *mapStateV5) next() migrateable {
|
|
v6 := NewMapState()
|
|
for _, v := range st.PinMap {
|
|
v6.Add(context.Background(), v.ToPin())
|
|
}
|
|
return v6.(*MapState)
|
|
}
|
|
|
|
// Last time we use this migration approach.
|
|
func (st *MapState) next() migrateable { return nil }
|
|
func (st *MapState) unmarshal(r io.Reader) error {
|
|
return st.dst.Unmarshal(r)
|
|
}
|
|
|
|
// Migrate code
|
|
|
|
func finalCopy(st *MapState, internal *MapState) {
|
|
st.dst = internal.dst
|
|
}
|
|
|
|
func (st *MapState) migrateFrom(version int, snap io.Reader) error {
|
|
var m, next migrateable
|
|
switch version {
|
|
case 1:
|
|
var mst1 mapStateV1
|
|
m = &mst1
|
|
case 2:
|
|
var mst2 mapStateV2
|
|
m = &mst2
|
|
case 3:
|
|
var mst3 mapStateV3
|
|
m = &mst3
|
|
case 4:
|
|
var mst4 mapStateV4
|
|
m = &mst4
|
|
case 5:
|
|
var mst5 mapStateV5
|
|
m = &mst5
|
|
default:
|
|
return errors.New("version migration not supported")
|
|
}
|
|
|
|
err := m.unmarshal(snap)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for {
|
|
next = m.next()
|
|
if next == nil {
|
|
mst6, ok := m.(*MapState)
|
|
if !ok {
|
|
return errors.New("migration ended prematurely")
|
|
}
|
|
finalCopy(st, mst6)
|
|
return nil
|
|
}
|
|
m = next
|
|
}
|
|
}
|