cascade/ipfs-cluster
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #663 from roignpar/issue_656

Browse Source 
Read config values from env on init command
This commit is contained in:
Hector Sanjuan 2019-02-19 17:48:47 +00:00 committed by GitHub
commit 3059ab387a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 582 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
api/ipfsproxy/config.go
View File

@ -82,7 +82,7 @@ type Config struct {
type jsonConfig struct {
ListenMultiaddress string `json:"listen_multiaddress"`
NodeMultiaddress string `json:"node_multiaddress"`
NodeHTTPS string `json:"node_https,omitempty"`
NodeHTTPS bool `json:"node_https,omitempty"`
ReadTimeout string `json:"read_timeout"`
ReadHeaderTimeout string `json:"read_header_timeout"`
@ -154,6 +154,22 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg, err := cfg.toJSONConfig()
if err != nil {
return err
}
err = envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have sensible values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -210,12 +226,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
return fmt.Errorf("error setting config to default values: %s", err)
}
// override json config with env var
err = envconfig.Process("cluster_ipfsproxy", jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
proxyAddr, err := ma.NewMultiaddr(jcfg.ListenMultiaddress)
if err != nil {
return fmt.Errorf("error parsing proxy listen_multiaddress: %s", err)
@ -251,6 +265,16 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg, err := cfg.toJSONConfig()
if err != nil {
return
}
raw, err = config.DefaultJSONMarshal(jcfg)
return
}
func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
// Multiaddress String() may panic
defer func() {
if r := recover(); r != nil {
@ -258,7 +282,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
}
}()
jcfg := &jsonConfig{}
jcfg = &jsonConfig{}
// Set all configuration fields
jcfg.ListenMultiaddress = cfg.ListenAddr.String()
@ -267,11 +291,11 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg.ReadHeaderTimeout = cfg.ReadHeaderTimeout.String()
jcfg.WriteTimeout = cfg.WriteTimeout.String()
jcfg.IdleTimeout = cfg.IdleTimeout.String()
jcfg.NodeHTTPS = cfg.NodeHTTPS
jcfg.ExtractHeadersExtra = cfg.ExtractHeadersExtra
jcfg.ExtractHeadersPath = cfg.ExtractHeadersPath
jcfg.ExtractHeadersTTL = cfg.ExtractHeadersTTL.String()
raw, err = config.DefaultJSONMarshal(jcfg)
return
}

13
api/ipfsproxy/config_test.go
View File

@ -2,7 +2,9 @@ package ipfsproxy
import (
"encoding/json"
"os"
"testing"
"time"
)
var cfgJSON = []byte(`
@ -125,3 +127,14 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_IPFSPROXY_IDLETIMEOUT", "22s")
cfg := &Config{}
cfg.Default()
cfg.ApplyEnvVars()
if cfg.IdleTimeout != 22*time.Second {
t.Error("failed to override idle_timeout with env var")
}
}

39
api/rest/config.go
View File

@ -181,6 +181,22 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg, err := cfg.toJSONConfig()
if err != nil {
return err
}
err = envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate makes sure that all fields in this Config have
// working values, at least in appearance.
func (cfg *Config) Validate() error {
@ -230,13 +246,11 @@ func (cfg *Config) LoadJSON(raw []byte) error {
cfg.Default()
// override json config with env var
err = envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
err = cfg.loadHTTPOptions(jcfg)
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
err := cfg.loadHTTPOptions(jcfg)
if err != nil {
return err
}
@ -361,6 +375,16 @@ func (cfg *Config) loadLibp2pOptions(jcfg *jsonConfig) error {
// ToJSON produce a human-friendly JSON representation of the Config
// object.
func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg, err := cfg.toJSONConfig()
if err != nil {
return
}
raw, err = config.DefaultJSONMarshal(jcfg)
return
}
func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
// Multiaddress String() may panic
defer func() {
if r := recover(); r != nil {
@ -368,7 +392,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
}
}()
jcfg := &jsonConfig{
jcfg = &jsonConfig{
HTTPListenMultiaddress: cfg.HTTPListenAddr.String(),
SSLCertFile: cfg.pathSSLCertFile,
SSLKeyFile: cfg.pathSSLKeyFile,
@ -400,7 +424,6 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg.Libp2pListenMultiaddress = cfg.Libp2pListenAddr.String()
}
raw, err = config.DefaultJSONMarshal(jcfg)
return
}

5
api/rest/config_test.go
View File

@ -110,14 +110,15 @@ func TestLoadJSON(t *testing.T) {
}
}
func TestLoadJSONEnvConfig(t *testing.T) {
func TestApplyEnvVars(t *testing.T) {
username := "admin"
password := "thisaintmypassword"
user1 := "user1"
user1pass := "user1passwd"
os.Setenv("CLUSTER_RESTAPI_BASICAUTHCREDS", username+":"+password+","+user1+":"+user1pass)
cfg := &Config{}
err := cfg.LoadJSON(cfgJSON)
cfg.Default()
err := cfg.ApplyEnvVars()
if err != nil {
t.Fatal(err)
}

63
cluster_config.go
View File

@ -186,6 +186,22 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg, err := cfg.toConfigJSON()
if err != nil {
return err
}
err = envconfig.Process(cfg.ConfigKey(), jcfg)
if err != nil {
return err
}
return cfg.applyConfigJSON(jcfg)
}
// Validate will check that the values of this config
// seem to be working ones.
func (cfg *Config) Validate() error {
@ -305,20 +321,10 @@ for more information.`)
return errors.New("cluster.Peers and cluster.Bootstrap keys have been deprecated")
}
// override json config with env var
err = envconfig.Process(cfg.ConfigKey(), jcfg)
if err != nil {
return err
}
parseDuration := func(txt string) time.Duration {
d, _ := time.ParseDuration(txt)
if txt != "" && d == 0 {
logger.Warningf("%s is not a valid duration. Default will be used", txt)
}
return d
}
return cfg.applyConfigJSON(jcfg)
}
func (cfg *Config) applyConfigJSON(jcfg *configJSON) error {
config.SetIfNotDefault(jcfg.PeerstoreFile, &cfg.PeerstoreFile)
id, err := peer.IDB58Decode(jcfg.ID)
@ -365,15 +371,15 @@ for more information.`)
config.SetIfNotDefault(rplMin, &cfg.ReplicationFactorMin)
config.SetIfNotDefault(rplMax, &cfg.ReplicationFactorMax)
stateSyncInterval := parseDuration(jcfg.StateSyncInterval)
ipfsSyncInterval := parseDuration(jcfg.IPFSSyncInterval)
monitorPingInterval := parseDuration(jcfg.MonitorPingInterval)
peerWatchInterval := parseDuration(jcfg.PeerWatchInterval)
config.SetIfNotDefault(stateSyncInterval, &cfg.StateSyncInterval)
config.SetIfNotDefault(ipfsSyncInterval, &cfg.IPFSSyncInterval)
config.SetIfNotDefault(monitorPingInterval, &cfg.MonitorPingInterval)
config.SetIfNotDefault(peerWatchInterval, &cfg.PeerWatchInterval)
err = config.ParseDurations("cluster",
&config.DurationOpt{Duration: jcfg.StateSyncInterval, Dst: &cfg.StateSyncInterval, Name: "state_sync_interval"},
&config.DurationOpt{Duration: jcfg.IPFSSyncInterval, Dst: &cfg.IPFSSyncInterval, Name: "ipfs_sync_interval"},
&config.DurationOpt{Duration: jcfg.MonitorPingInterval, Dst: &cfg.MonitorPingInterval, Name: "monitor_ping_interval"},
&config.DurationOpt{Duration: jcfg.PeerWatchInterval, Dst: &cfg.PeerWatchInterval, Name: "peer_watch_interval"},
)
if err != nil {
return err
}
cfg.LeaveOnShutdown = jcfg.LeaveOnShutdown
cfg.DisableRepinning = jcfg.DisableRepinning
@ -383,6 +389,16 @@ for more information.`)
// ToJSON generates a human-friendly version of Config.
func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg, err := cfg.toConfigJSON()
if err != nil {
return
}
raw, err = json.MarshalIndent(jcfg, "", " ")
return
}
func (cfg *Config) toConfigJSON() (jcfg *configJSON, err error) {
// Multiaddress String() may panic
defer func() {
if r := recover(); r != nil {
@ -390,7 +406,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
}
}()
jcfg := &configJSON{}
jcfg = &configJSON{}
// Private Key
pkeyBytes, err := cfg.PrivateKey.Bytes()
@ -415,7 +431,6 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg.DisableRepinning = cfg.DisableRepinning
jcfg.PeerstoreFile = cfg.PeerstoreFile
raw, err = json.MarshalIndent(jcfg, "", " ")
return
}

19
cluster_config_test.go
View File

@ -187,15 +187,6 @@ func TestLoadJSON(t *testing.T) {
t.Error("expected default replication factors")
}
})
t.Run("env var override", func(t *testing.T) {
os.Setenv("CLUSTER_PEERNAME", "envsetpeername")
cfg := &Config{}
cfg.LoadJSON(ccfgTestJSON)
if cfg.Peername != "envsetpeername" {
t.Fatal("failed to override peername with env var")
}
})
}
func TestToJSON(t *testing.T) {
@ -220,6 +211,16 @@ func TestDefault(t *testing.T) {
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_PEERNAME", "envsetpeername")
cfg := &Config{}
cfg.Default()
cfg.ApplyEnvVars()
if cfg.Peername != "envsetpeername" {
t.Fatal("failed to override peername with env var")
}
}
func TestValidate(t *testing.T) {
cfg := &Config{}
cfg.Default()

2
cmd/ipfs-cluster-service/daemon.go
View File

@ -69,7 +69,7 @@ func daemon(c *cli.Context) error {
// always wait for configuration to be saved
defer cfgMgr.Shutdown()
err = cfgMgr.LoadJSONFromFile(configPath)
err = cfgMgr.LoadJSONFileAndEnv(configPath)
checkErr("loading configuration", err)
if c.Bool("stats") {

19
cmd/ipfs-cluster-service/main.go
View File

@ -240,6 +240,9 @@ configuration.
err := cfgMgr.Default()
checkErr("generating default configuration", err)
err = cfgMgr.ApplyEnvVars()
checkErr("applying environment variables to configuration", err)
// Set user secret
if userSecretDefined {
cfgs.clusterCfg.Secret = userSecret
@ -439,7 +442,7 @@ the mth data folder (m currently defaults to 5)
}
cfgMgr, cfgs := makeConfigs()
err = cfgMgr.LoadJSONFromFile(configPath)
err = cfgMgr.LoadJSONFileAndEnv(configPath)
checkErr("reading configuration", err)
err = cleanupState(cfgs.consensusCfg)
@ -501,18 +504,14 @@ func setupDebug() {
}
func userProvidedSecret(enterSecret bool) ([]byte, bool) {
var secret string
if enterSecret {
secret = promptUser("Enter cluster secret (32-byte hex string): ")
} else if envSecret, envSecretDefined := os.LookupEnv("CLUSTER_SECRET"); envSecretDefined {
secret = envSecret
} else {
return nil, false
secret := promptUser("Enter cluster secret (32-byte hex string): ")
decodedSecret, err := ipfscluster.DecodeClusterSecret(secret)
checkErr("parsing user-provided secret", err)
return decodedSecret, true
}
decodedSecret, err := ipfscluster.DecodeClusterSecret(secret)
checkErr("parsing user-provided secret", err)
return decodedSecret, true
return nil, false
}
func promptUser(msg string) string {

6
cmd/ipfs-cluster-service/state.go
View File

@ -34,7 +34,7 @@ func upgrade(ctx context.Context) error {
cfgMgr, cfgs := makeConfigs()
err = cfgMgr.LoadJSONFromFile(configPath)
err = cfgMgr.LoadJSONFileAndEnv(configPath)
if err != nil {
return err
}
@ -65,7 +65,7 @@ func restoreStateFromDisk(ctx context.Context) (*mapstate.MapState, bool, error)
cfgMgr, cfgs := makeConfigs()
err := cfgMgr.LoadJSONFromFile(configPath)
err := cfgMgr.LoadJSONFileAndEnv(configPath)
if err != nil {
return nil, false, err
}
@ -108,7 +108,7 @@ func stateImport(ctx context.Context, r io.Reader) error {
cfgMgr, cfgs := makeConfigs()
err := cfgMgr.LoadJSONFromFile(configPath)
err := cfgMgr.LoadJSONFileAndEnv(configPath)
if err != nil {
return err
}

37
config/config.go
View File

@ -34,6 +34,8 @@ type ComponentConfig interface {
ToJSON() ([]byte, error)
// Sets default working values
Default() error
// Sets values from environment variables
ApplyEnvVars() error
// Allows this component to work under a subfolder
SetBaseDir(string)
// Checks that the configuration is valid
@ -225,6 +227,30 @@ func (cfg *Manager) Default() error {
return nil
}
// ApplyEnvVars overrides configuration fields with any values found
// in environment variables.
func (cfg *Manager) ApplyEnvVars() error {
for _, section := range cfg.sections {
for k, compcfg := range section {
logger.Debugf("applying environment variables conf for %s", k)
err := compcfg.ApplyEnvVars()
if err != nil {
return err
}
}
}
if cfg.clusterConfig != nil {
logger.Debugf("applying environment variables conf for cluster")
err := cfg.clusterConfig.ApplyEnvVars()
if err != nil {
return err
}
}
return nil
}
// RegisterComponent lets the Manager load and save component configurations
func (cfg *Manager) RegisterComponent(t SectionType, ccfg ComponentConfig) {
cfg.wg.Add(1)
@ -296,6 +322,17 @@ func (cfg *Manager) LoadJSONFromFile(path string) error {
return err
}
// LoadJSONFileAndEnv calls LoadJSONFromFile followed by ApplyEnvVars,
// reading and parsing a Configuration file and then overriding fields
// with any values found in environment variables.
func (cfg *Manager) LoadJSONFileAndEnv(path string) error {
if err := cfg.LoadJSONFromFile(path); err != nil {
return err
}
return cfg.ApplyEnvVars()
}
// LoadJSON parses configurations for all registered components,
// In order to work, component configurations must have been registered
// beforehand with RegisterComponent.

4
config/config_test.go
View File

@ -30,6 +30,10 @@ func (m *mockCfg) Default() error {
return nil
}
func (m *mockCfg) ApplyEnvVars() error {
return nil
}
func (m *mockCfg) Validate() error {
return nil
}

29
consensus/raft/config.go
View File

@ -9,6 +9,7 @@ import (
"github.com/ipfs/ipfs-cluster/api"
"github.com/ipfs/ipfs-cluster/config"
"github.com/kelseyhightower/envconfig"
hraft "github.com/hashicorp/raft"
peer "github.com/libp2p/go-libp2p-peer"
@ -17,6 +18,7 @@ import (
// ConfigKey is the default configuration key for holding this component's
// configuration section.
var configKey = "raft"
var envConfigKey = "cluster_raft"
// Configuration defaults
var (
@ -185,6 +187,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
parseDuration := func(txt string) time.Duration {
d, _ := time.ParseDuration(txt)
if txt != "" && d == 0 {
@ -230,7 +236,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON returns the pretty JSON representation of a Config.
func (cfg *Config) ToJSON() ([]byte, error) {
jcfg := &jsonConfig{
jcfg := cfg.toJSONConfig()
return config.DefaultJSONMarshal(jcfg)
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
DataFolder: cfg.DataFolder,
InitPeerset: api.PeersToStrings(cfg.InitPeerset),
WaitForLeaderTimeout: cfg.WaitForLeaderTimeout.String(),
@ -247,8 +259,6 @@ func (cfg *Config) ToJSON() ([]byte, error) {
SnapshotThreshold: cfg.RaftConfig.SnapshotThreshold,
LeaderLeaseTimeout: cfg.RaftConfig.LeaderLeaseTimeout.String(),
}
return config.DefaultJSONMarshal(jcfg)
}
// Default initializes this configuration with working defaults.
@ -272,6 +282,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// GetDataFolder returns the Raft data folder that we are using.
func (cfg *Config) GetDataFolder() string {
if cfg.DataFolder == "" {

12
consensus/raft/config_test.go
View File

@ -2,6 +2,7 @@ package raft
import (
"encoding/json"
"os"
"testing"
hraft "github.com/hashicorp/raft"
@ -106,3 +107,14 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_RAFT_COMMITRETRIES", "300")
cfg := &Config{}
cfg.Default()
cfg.ApplyEnvVars()
if cfg.CommitRetries != 300 {
t.Fatal("failed to override commit_retries with env var")
}
}

33
informer/disk/config.go
View File

@ -6,9 +6,11 @@ import (
"time"
"github.com/ipfs/ipfs-cluster/config"
"github.com/kelseyhightower/envconfig"
)
const configKey = "disk"
const envConfigKey = "cluster_disk"
// Default values for disk Config
const (
@ -53,6 +55,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -76,6 +91,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
return err
}
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
t, _ := time.ParseDuration(jcfg.MetricTTL)
cfg.MetricTTL = t
@ -94,11 +115,15 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a JSON-formatted human-friendly representation of this
// Config.
func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg := &jsonConfig{}
jcfg.MetricTTL = cfg.MetricTTL.String()
jcfg.Type = cfg.Type.String()
jcfg := cfg.toJSONConfig()
raw, err = config.DefaultJSONMarshal(jcfg)
return
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
MetricTTL: cfg.MetricTTL.String(),
Type: cfg.Type.String(),
}
}

12
informer/disk/config_test.go
View File

@ -2,7 +2,9 @@ package disk
import (
"encoding/json"
"os"
"testing"
"time"
)
var cfgJSON = []byte(`
@ -81,3 +83,13 @@ func TestDefault(t *testing.T) {
t.Fatal("MetricRepoSize is a valid type")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_DISK_METRICTTL", "22s")
cfg := &Config{}
cfg.ApplyEnvVars()
if cfg.MetricTTL != 22*time.Second {
t.Fatal("failed to override metric_ttl with env var")
}
}

31
informer/numpin/config.go
View File

@ -6,9 +6,11 @@ import (
"time"
"github.com/ipfs/ipfs-cluster/config"
"github.com/kelseyhightower/envconfig"
)
const configKey = "numpin"
const envConfigKey = "cluster_numpin"
// These are the default values for a Config.
const (
@ -38,6 +40,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this configuration have
// sensible values.
func (cfg *Config) Validate() error {
@ -56,6 +71,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
return err
}
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
t, _ := time.ParseDuration(jcfg.MetricTTL)
cfg.MetricTTL = t
@ -64,9 +85,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() ([]byte, error) {
jcfg := &jsonConfig{}
jcfg.MetricTTL = cfg.MetricTTL.String()
jcfg := cfg.toJSONConfig()
return config.DefaultJSONMarshal(jcfg)
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
MetricTTL: cfg.MetricTTL.String(),
}
}

12
informer/numpin/config_test.go
View File

@ -2,7 +2,9 @@ package numpin
import (
"encoding/json"
"os"
"testing"
"time"
)
var cfgJSON = []byte(`
@ -55,3 +57,13 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_NUMPIN_METRICTTL", "22s")
cfg := &Config{}
cfg.ApplyEnvVars()
if cfg.MetricTTL != 22*time.Second {
t.Fatal("failed to override metric_ttl with env var")
}
}

36
ipfsconn/ipfshttp/config.go
View File

@ -6,12 +6,15 @@ import (
"fmt"
"time"
"github.com/kelseyhightower/envconfig"
"github.com/ipfs/ipfs-cluster/config"
ma "github.com/multiformats/go-multiaddr"
)
const configKey = "ipfshttp"
const envConfigKey = "cluster_ipfshttp"
// Default values for Config.
const (
@ -89,6 +92,22 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg, err := cfg.toJSONConfig()
if err != nil {
return err
}
err = envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have sensible values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -133,6 +152,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
nodeAddr, err := ma.NewMultiaddr(jcfg.NodeMultiaddress)
if err != nil {
return fmt.Errorf("error parsing ipfs_node_multiaddress: %s", err)
@ -158,6 +181,16 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg, err := cfg.toJSONConfig()
if err != nil {
return
}
raw, err = config.DefaultJSONMarshal(jcfg)
return
}
func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
// Multiaddress String() may panic
defer func() {
if r := recover(); r != nil {
@ -165,7 +198,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
}
}()
jcfg := &jsonConfig{}
jcfg = &jsonConfig{}
// Set all configuration fields
jcfg.NodeMultiaddress = cfg.NodeAddr.String()
@ -175,6 +208,5 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
jcfg.PinTimeout = cfg.PinTimeout.String()
jcfg.UnpinTimeout = cfg.UnpinTimeout.String()
raw, err = config.DefaultJSONMarshal(jcfg)
return
}

13
ipfsconn/ipfshttp/config_test.go
View File

@ -2,7 +2,9 @@ package ipfshttp
import (
"encoding/json"
"os"
"testing"
"time"
)
var cfgJSON = []byte(`
@ -59,3 +61,14 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVar(t *testing.T) {
os.Setenv("CLUSTER_IPFSHTTP_PINTIMEOUT", "22m")
cfg := &Config{}
cfg.Default()
cfg.ApplyEnvVars()
if cfg.PinTimeout != 22*time.Minute {
t.Fatal("failed to override pin_timeout with env var")
}
}

31
monitor/basic/config.go
View File

@ -6,9 +6,11 @@ import (
"time"
"github.com/ipfs/ipfs-cluster/config"
"github.com/kelseyhightower/envconfig"
)
const configKey = "monbasic"
const envConfigKey = "cluster_monbasic"
// Default values for this Config.
const (
@ -37,6 +39,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -56,6 +71,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
return err
}
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
interval, _ := time.ParseDuration(jcfg.CheckInterval)
cfg.CheckInterval = interval
@ -64,9 +85,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() ([]byte, error) {
jcfg := &jsonConfig{}
jcfg.CheckInterval = cfg.CheckInterval.String()
jcfg := cfg.toJSONConfig()
return json.MarshalIndent(jcfg, "", " ")
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
CheckInterval: cfg.CheckInterval.String(),
}
}

12
monitor/basic/config_test.go
View File

@ -2,7 +2,9 @@ package basic
import (
"encoding/json"
"os"
"testing"
"time"
)
var cfgJSON = []byte(`
@ -55,3 +57,13 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_MONBASIC_CHECKINTERVAL", "22s")
cfg := &Config{}
cfg.ApplyEnvVars()
if cfg.CheckInterval != 22*time.Second {
t.Fatal("failed to override check_interval with env var")
}
}

31
monitor/pubsubmon/config.go
View File

@ -6,9 +6,11 @@ import (
"time"
"github.com/ipfs/ipfs-cluster/config"
"github.com/kelseyhightower/envconfig"
)
const configKey = "pubsubmon"
const envConfigKey = "cluster_pubsubmon"
// Default values for this Config.
const (
@ -37,6 +39,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -56,6 +71,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
return err
}
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
interval, _ := time.ParseDuration(jcfg.CheckInterval)
cfg.CheckInterval = interval
@ -64,9 +85,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() ([]byte, error) {
jcfg := &jsonConfig{}
jcfg.CheckInterval = cfg.CheckInterval.String()
jcfg := cfg.toJSONConfig()
return json.MarshalIndent(jcfg, "", " ")
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
CheckInterval: cfg.CheckInterval.String(),
}
}

12
monitor/pubsubmon/config_test.go
View File

@ -2,7 +2,9 @@ package pubsubmon
import (
"encoding/json"
"os"
"testing"
"time"
)
var cfgJSON = []byte(`
@ -55,3 +57,13 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_PUBSUBMON_CHECKINTERVAL", "22s")
cfg := &Config{}
cfg.ApplyEnvVars()
if cfg.CheckInterval != 22*time.Second {
t.Fatal("failed to override check_interval with env var")
}
}

69
observations/config.go
View File

@ -14,7 +14,8 @@ import (
const metricsConfigKey = "metrics"
const tracingConfigKey = "tracing"
const envConfigKey = "cluster_observations"
const metricsEnvConfigKey = "cluster_metrics"
const tracingEnvConfigKey = "cluster_tracing"
// Default values for this Config.
const (
@ -58,6 +59,19 @@ func (cfg *MetricsConfig) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *MetricsConfig) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(metricsEnvConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *MetricsConfig) Validate() error {
@ -84,13 +98,11 @@ func (cfg *MetricsConfig) LoadJSON(raw []byte) error {
cfg.Default()
// override json config with env var
err = envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
err = cfg.loadMetricsOptions(jcfg)
func (cfg *MetricsConfig) applyJSONConfig(jcfg *jsonMetricsConfig) error {
err := cfg.loadMetricsOptions(jcfg)
if err != nil {
return err
}
@ -118,13 +130,17 @@ func (cfg *MetricsConfig) loadMetricsOptions(jcfg *jsonMetricsConfig) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *MetricsConfig) ToJSON() ([]byte, error) {
jcfg := &jsonMetricsConfig{
jcfg := cfg.toJSONConfig()
return config.DefaultJSONMarshal(jcfg)
}
func (cfg *MetricsConfig) toJSONConfig() *jsonMetricsConfig {
return &jsonMetricsConfig{
EnableStats: cfg.EnableStats,
PrometheusEndpoint: cfg.PrometheusEndpoint.String(),
ReportingInterval: cfg.ReportingInterval.String(),
}
return config.DefaultJSONMarshal(jcfg)
}
// TracingConfig configures tracing.
@ -159,6 +175,19 @@ func (cfg *TracingConfig) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *TracingConfig) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(tracingEnvConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *TracingConfig) Validate() error {
@ -185,13 +214,11 @@ func (cfg *TracingConfig) LoadJSON(raw []byte) error {
cfg.Default()
// override json config with env var
err = envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
err = cfg.loadTracingOptions(jcfg)
func (cfg *TracingConfig) applyJSONConfig(jcfg *jsonTracingConfig) error {
err := cfg.loadTracingOptions(jcfg)
if err != nil {
return err
}
@ -214,12 +241,16 @@ func (cfg *TracingConfig) loadTracingOptions(jcfg *jsonTracingConfig) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *TracingConfig) ToJSON() ([]byte, error) {
jcfg := &jsonTracingConfig{
jcfg := cfg.toJSONConfig()
return config.DefaultJSONMarshal(jcfg)
}
func (cfg *TracingConfig) toJSONConfig() *jsonTracingConfig {
return &jsonTracingConfig{
EnableTracing: cfg.EnableTracing,
JaegerAgentEndpoint: cfg.JaegerAgentEndpoint.String(),
SamplingProb: cfg.SamplingProb,
ServiceName: cfg.ServiceName,
}
return config.DefaultJSONMarshal(jcfg)
}

26
observations/config_test.go Normal file
View File

@ -0,0 +1,26 @@
package observations
import (
"os"
"testing"
)
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_METRICS_ENABLESTATS", "true")
mcfg := &MetricsConfig{}
mcfg.Default()
mcfg.ApplyEnvVars()
if !mcfg.EnableStats {
t.Fatal("failed to override enable_stats with env var")
}
os.Setenv("CLUSTER_TRACING_ENABLETRACING", "true")
tcfg := &TracingConfig{}
tcfg.Default()
tcfg.ApplyEnvVars()
if !tcfg.EnableTracing {
t.Fatal("failed to override enable_tracing with env var")
}
}

32
pintracker/maptracker/config.go
View File

@ -4,10 +4,13 @@ import (
"encoding/json"
"errors"
"github.com/kelseyhightower/envconfig"
"github.com/ipfs/ipfs-cluster/config"
)
const configKey = "maptracker"
const envConfigKey = "cluster_maptracker"
// Default values for this Config.
const (
@ -44,6 +47,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -69,6 +85,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
config.SetIfNotDefault(jcfg.MaxPinQueueSize, &cfg.MaxPinQueueSize)
config.SetIfNotDefault(jcfg.ConcurrentPins, &cfg.ConcurrentPins)
@ -77,10 +97,14 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() ([]byte, error) {
jcfg := &jsonConfig{}
jcfg.MaxPinQueueSize = cfg.MaxPinQueueSize
jcfg.ConcurrentPins = cfg.ConcurrentPins
jcfg := cfg.toJSONConfig()
return config.DefaultJSONMarshal(jcfg)
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
MaxPinQueueSize: cfg.MaxPinQueueSize,
ConcurrentPins: cfg.ConcurrentPins,
}
}

11
pintracker/maptracker/config_test.go
View File

@ -2,6 +2,7 @@ package maptracker
import (
"encoding/json"
"os"
"testing"
)
@ -59,3 +60,13 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_MAPTRACKER_CONCURRENTPINS", "22")
cfg := &Config{}
cfg.ApplyEnvVars()
if cfg.ConcurrentPins != 22 {
t.Fatal("failed to override concurrent_pins with env var")
}
}

32
pintracker/stateless/config.go
View File

@ -4,10 +4,13 @@ import (
"encoding/json"
"errors"
"github.com/kelseyhightower/envconfig"
"github.com/ipfs/ipfs-cluster/config"
)
const configKey = "stateless"
const envConfigKey = "cluster_stateless"
// Default values for this Config.
const (
@ -44,6 +47,19 @@ func (cfg *Config) Default() error {
return nil
}
// ApplyEnvVars fills in any Config fields found
// as environment variables.
func (cfg *Config) ApplyEnvVars() error {
jcfg := cfg.toJSONConfig()
err := envconfig.Process(envConfigKey, jcfg)
if err != nil {
return err
}
return cfg.applyJSONConfig(jcfg)
}
// Validate checks that the fields of this Config have working values,
// at least in appearance.
func (cfg *Config) Validate() error {
@ -69,6 +85,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
cfg.Default()
return cfg.applyJSONConfig(jcfg)
}
func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
config.SetIfNotDefault(jcfg.MaxPinQueueSize, &cfg.MaxPinQueueSize)
config.SetIfNotDefault(jcfg.ConcurrentPins, &cfg.ConcurrentPins)
@ -77,10 +97,14 @@ func (cfg *Config) LoadJSON(raw []byte) error {
// ToJSON generates a human-friendly JSON representation of this Config.
func (cfg *Config) ToJSON() ([]byte, error) {
jcfg := &jsonConfig{}
jcfg.MaxPinQueueSize = cfg.MaxPinQueueSize
jcfg.ConcurrentPins = cfg.ConcurrentPins
jcfg := cfg.toJSONConfig()
return config.DefaultJSONMarshal(jcfg)
}
func (cfg *Config) toJSONConfig() *jsonConfig {
return &jsonConfig{
MaxPinQueueSize: cfg.MaxPinQueueSize,
ConcurrentPins: cfg.ConcurrentPins,
}
}

11
pintracker/stateless/config_test.go
View File

@ -2,6 +2,7 @@ package stateless
import (
"encoding/json"
"os"
"testing"
)
@ -59,3 +60,13 @@ func TestDefault(t *testing.T) {
t.Fatal("expected error validating")
}
}
func TestApplyEnvVars(t *testing.T) {
os.Setenv("CLUSTER_STATELESS_CONCURRENTPINS", "22")
cfg := &Config{}
cfg.ApplyEnvVars()
if cfg.ConcurrentPins != 22 {
t.Fatal("failed to override concurrent_pins with env var")
}
}