fb4812ec79
This fixes #326. It adds a new `pin_method` configuration option to the `ipfshttp` component allows to configure it to perform `refs -r <cid>` before the `pin/add` call. By fetching content before pinning, we don't have a global lock in place, and we can have several pin-requests to ipfs in parallel. It also adds a `concurrent_pins` option to the pin tracker, which launches more pin workers so it can potentially trigger more pins at the same time. This is a minimal intervention in the pintracker as #308 is still pending. Documentation for the configuration file has been updated. License: MIT Signed-off-by: Hector Sanjuan <code@hector.link>
119 lines
3.5 KiB
Go
119 lines
3.5 KiB
Go
package maptracker
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"time"
|
|
|
|
"github.com/ipfs/ipfs-cluster/config"
|
|
)
|
|
|
|
const configKey = "maptracker"
|
|
|
|
// Default values for this Config.
|
|
const (
|
|
DefaultPinningTimeout = 60 * time.Minute
|
|
DefaultUnpinningTimeout = 5 * time.Minute
|
|
DefaultMaxPinQueueSize = 4096
|
|
DefaultConcurrentPins = 1
|
|
)
|
|
|
|
// Config allows to initialize a Monitor and customize some parameters.
|
|
type Config struct {
|
|
config.Saver
|
|
|
|
// PinningTimeout specifies how long to wait before a pinning state becomes a pin error
|
|
PinningTimeout time.Duration
|
|
// UnpinningTimeout specifies how long to wait before an unpinning state becomes a pin error
|
|
UnpinningTimeout time.Duration
|
|
// MaxPinQueueSize specifies how many pin or unpin requests we can hold in the queue
|
|
// If higher, they will automatically marked with an error.
|
|
MaxPinQueueSize int
|
|
// ConcurrentPins specifies how many pin requests can be sent to the ipfs
|
|
// daemon in parallel. If the pinning method is "refs", it might increase
|
|
// speed. Unpin requests are always processed one by one.
|
|
ConcurrentPins int
|
|
}
|
|
|
|
type jsonConfig struct {
|
|
PinningTimeout string `json:"pinning_timeout"`
|
|
UnpinningTimeout string `json:"unpinning_timeout"`
|
|
MaxPinQueueSize int `json:"max_pin_queue_size"`
|
|
ConcurrentPins int `json:"concurrent_pins"`
|
|
}
|
|
|
|
// ConfigKey provides a human-friendly identifier for this type of Config.
|
|
func (cfg *Config) ConfigKey() string {
|
|
return configKey
|
|
}
|
|
|
|
// Default sets the fields of this Config to sensible values.
|
|
func (cfg *Config) Default() error {
|
|
cfg.PinningTimeout = DefaultPinningTimeout
|
|
cfg.UnpinningTimeout = DefaultUnpinningTimeout
|
|
cfg.MaxPinQueueSize = DefaultMaxPinQueueSize
|
|
cfg.ConcurrentPins = DefaultConcurrentPins
|
|
return nil
|
|
}
|
|
|
|
// Validate checks that the fields of this Config have working values,
|
|
// at least in appearance.
|
|
func (cfg *Config) Validate() error {
|
|
if cfg.PinningTimeout <= 0 {
|
|
return errors.New("maptracker.pinning_timeout too low")
|
|
}
|
|
if cfg.UnpinningTimeout <= 0 {
|
|
return errors.New("maptracker.unpinning_timeout too low")
|
|
}
|
|
if cfg.MaxPinQueueSize <= 0 {
|
|
return errors.New("maptracker.max_pin_queue_size too low")
|
|
}
|
|
|
|
if cfg.ConcurrentPins <= 0 {
|
|
return errors.New("maptracker.concurrent_pins is too low")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// LoadJSON sets the fields of this Config to the values defined by the JSON
|
|
// representation of it, as generated by ToJSON.
|
|
func (cfg *Config) LoadJSON(raw []byte) error {
|
|
jcfg := &jsonConfig{}
|
|
err := json.Unmarshal(raw, jcfg)
|
|
if err != nil {
|
|
logger.Error("Error unmarshaling basic monitor config")
|
|
return err
|
|
}
|
|
|
|
cfg.Default()
|
|
|
|
parseDuration := func(txt string) time.Duration {
|
|
d, _ := time.ParseDuration(txt)
|
|
if txt != "" && d == 0 {
|
|
logger.Warningf("%s is not a valid duration. Default will be used", txt)
|
|
}
|
|
return d
|
|
}
|
|
|
|
pinningTimeo := parseDuration(jcfg.PinningTimeout)
|
|
unpinningTimeo := parseDuration(jcfg.UnpinningTimeout)
|
|
config.SetIfNotDefault(pinningTimeo, &cfg.PinningTimeout)
|
|
config.SetIfNotDefault(unpinningTimeo, &cfg.UnpinningTimeout)
|
|
config.SetIfNotDefault(jcfg.MaxPinQueueSize, &cfg.MaxPinQueueSize)
|
|
config.SetIfNotDefault(jcfg.ConcurrentPins, &cfg.ConcurrentPins)
|
|
|
|
return cfg.Validate()
|
|
}
|
|
|
|
// ToJSON generates a human-friendly JSON representation of this Config.
|
|
func (cfg *Config) ToJSON() ([]byte, error) {
|
|
jcfg := &jsonConfig{}
|
|
|
|
jcfg.PinningTimeout = cfg.PinningTimeout.String()
|
|
jcfg.UnpinningTimeout = cfg.UnpinningTimeout.String()
|
|
jcfg.MaxPinQueueSize = cfg.MaxPinQueueSize
|
|
jcfg.ConcurrentPins = cfg.ConcurrentPins
|
|
|
|
return config.DefaultJSONMarshal(jcfg)
|
|
}
|