Merge pull request #663 from roignpar/issue_656

Read config values from env on init command

api/ipfsproxy/config.go

@@ -82,7 +82,7 @@ type Config struct {
 type jsonConfig struct {
 	ListenMultiaddress string `json:"listen_multiaddress"`
 	NodeMultiaddress   string `json:"node_multiaddress"`
-	NodeHTTPS          string `json:"node_https,omitempty"`
+	NodeHTTPS          bool   `json:"node_https,omitempty"`
 
 	ReadTimeout       string `json:"read_timeout"`
 	ReadHeaderTimeout string `json:"read_header_timeout"`
@@ -154,6 +154,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have sensible values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -210,12 +226,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 		return fmt.Errorf("error setting config to default values: %s", err)
 	}
 
-	// override json config with env var
+	return cfg.applyJSONConfig(jcfg)
-	err = envconfig.Process("cluster_ipfsproxy", jcfg)
+}
-	if err != nil {
-		return err
-	}
 
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	proxyAddr, err := ma.NewMultiaddr(jcfg.ListenMultiaddress)
 	if err != nil {
 		return fmt.Errorf("error parsing proxy listen_multiaddress: %s", err)
@@ -251,6 +265,16 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return
+	}
+
+	raw, err = config.DefaultJSONMarshal(jcfg)
+	return
+}
+
+func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
@@ -258,7 +282,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 		}
 	}()
 
-	jcfg := &jsonConfig{}
+	jcfg = &jsonConfig{}
 
 	// Set all configuration fields
 	jcfg.ListenMultiaddress = cfg.ListenAddr.String()
@@ -267,11 +291,11 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 	jcfg.ReadHeaderTimeout = cfg.ReadHeaderTimeout.String()
 	jcfg.WriteTimeout = cfg.WriteTimeout.String()
 	jcfg.IdleTimeout = cfg.IdleTimeout.String()
+	jcfg.NodeHTTPS = cfg.NodeHTTPS
 
 	jcfg.ExtractHeadersExtra = cfg.ExtractHeadersExtra
 	jcfg.ExtractHeadersPath = cfg.ExtractHeadersPath
 	jcfg.ExtractHeadersTTL = cfg.ExtractHeadersTTL.String()
 
-	raw, err = config.DefaultJSONMarshal(jcfg)
 	return
 }

api/ipfsproxy/config_test.go

@@ -2,7 +2,9 @@ package ipfsproxy
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -125,3 +127,14 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_IPFSPROXY_IDLETIMEOUT", "22s")
+	cfg := &Config{}
+	cfg.Default()
+	cfg.ApplyEnvVars()
+
+	if cfg.IdleTimeout != 22*time.Second {
+		t.Error("failed to override idle_timeout with env var")
+	}
+}

api/rest/config.go

@@ -181,6 +181,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate makes sure that all fields in this Config have
 // working values, at least in appearance.
 func (cfg *Config) Validate() error {
@@ -230,13 +246,11 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
-	// override json config with env var
+	return cfg.applyJSONConfig(jcfg)
-	err = envconfig.Process(envConfigKey, jcfg)
+}
-	if err != nil {
-		return err
-	}
 
-	err = cfg.loadHTTPOptions(jcfg)
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
+	err := cfg.loadHTTPOptions(jcfg)
 	if err != nil {
 		return err
 	}
@@ -361,6 +375,16 @@ func (cfg *Config) loadLibp2pOptions(jcfg *jsonConfig) error {
 // ToJSON produce a human-friendly JSON representation of the Config
 // object.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return
+	}
+
+	raw, err = config.DefaultJSONMarshal(jcfg)
+	return
+}
+
+func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
@@ -368,7 +392,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 		}
 	}()
 
-	jcfg := &jsonConfig{
+	jcfg = &jsonConfig{
 		HTTPListenMultiaddress: cfg.HTTPListenAddr.String(),
 		SSLCertFile:            cfg.pathSSLCertFile,
 		SSLKeyFile:             cfg.pathSSLKeyFile,
@@ -400,7 +424,6 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 		jcfg.Libp2pListenMultiaddress = cfg.Libp2pListenAddr.String()
 	}
 
-	raw, err = config.DefaultJSONMarshal(jcfg)
 	return
 }
 

api/rest/config_test.go

@@ -110,14 +110,15 @@ func TestLoadJSON(t *testing.T) {
 	}
 }
 
-func TestLoadJSONEnvConfig(t *testing.T) {
+func TestApplyEnvVars(t *testing.T) {
 	username := "admin"
 	password := "thisaintmypassword"
 	user1 := "user1"
 	user1pass := "user1passwd"
 	os.Setenv("CLUSTER_RESTAPI_BASICAUTHCREDS", username+":"+password+","+user1+":"+user1pass)
 	cfg := &Config{}
-	err := cfg.LoadJSON(cfgJSON)
+	cfg.Default()
+	err := cfg.ApplyEnvVars()
 	if err != nil {
 		t.Fatal(err)
 	}

cluster_config.go

@@ -186,6 +186,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toConfigJSON()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(cfg.ConfigKey(), jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyConfigJSON(jcfg)
+}
+
 // Validate will check that the values of this config
 // seem to be working ones.
 func (cfg *Config) Validate() error {
@@ -305,20 +321,10 @@ for more information.`)
 		return errors.New("cluster.Peers and cluster.Bootstrap keys have been deprecated")
 	}
 
-	// override json config with env var
+	return cfg.applyConfigJSON(jcfg)
-	err = envconfig.Process(cfg.ConfigKey(), jcfg)
+}
-	if err != nil {
-		return err
-	}
-
-	parseDuration := func(txt string) time.Duration {
-		d, _ := time.ParseDuration(txt)
-		if txt != "" && d == 0 {
-			logger.Warningf("%s is not a valid duration. Default will be used", txt)
-		}
-		return d
-	}
 
+func (cfg *Config) applyConfigJSON(jcfg *configJSON) error {
 	config.SetIfNotDefault(jcfg.PeerstoreFile, &cfg.PeerstoreFile)
 
 	id, err := peer.IDB58Decode(jcfg.ID)
@@ -365,15 +371,15 @@ for more information.`)
 	config.SetIfNotDefault(rplMin, &cfg.ReplicationFactorMin)
 	config.SetIfNotDefault(rplMax, &cfg.ReplicationFactorMax)
 
-	stateSyncInterval := parseDuration(jcfg.StateSyncInterval)
+	err = config.ParseDurations("cluster",
-	ipfsSyncInterval := parseDuration(jcfg.IPFSSyncInterval)
+		&config.DurationOpt{Duration: jcfg.StateSyncInterval, Dst: &cfg.StateSyncInterval, Name: "state_sync_interval"},
-	monitorPingInterval := parseDuration(jcfg.MonitorPingInterval)
+		&config.DurationOpt{Duration: jcfg.IPFSSyncInterval, Dst: &cfg.IPFSSyncInterval, Name: "ipfs_sync_interval"},
-	peerWatchInterval := parseDuration(jcfg.PeerWatchInterval)
+		&config.DurationOpt{Duration: jcfg.MonitorPingInterval, Dst: &cfg.MonitorPingInterval, Name: "monitor_ping_interval"},
-
+		&config.DurationOpt{Duration: jcfg.PeerWatchInterval, Dst: &cfg.PeerWatchInterval, Name: "peer_watch_interval"},
-	config.SetIfNotDefault(stateSyncInterval, &cfg.StateSyncInterval)
+	)
-	config.SetIfNotDefault(ipfsSyncInterval, &cfg.IPFSSyncInterval)
+	if err != nil {
-	config.SetIfNotDefault(monitorPingInterval, &cfg.MonitorPingInterval)
+		return err
-	config.SetIfNotDefault(peerWatchInterval, &cfg.PeerWatchInterval)
+	}
 
 	cfg.LeaveOnShutdown = jcfg.LeaveOnShutdown
 	cfg.DisableRepinning = jcfg.DisableRepinning
@@ -383,6 +389,16 @@ for more information.`)
 
 // ToJSON generates a human-friendly version of Config.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toConfigJSON()
+	if err != nil {
+		return
+	}
+
+	raw, err = json.MarshalIndent(jcfg, "", "    ")
+	return
+}
+
+func (cfg *Config) toConfigJSON() (jcfg *configJSON, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
@@ -390,7 +406,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 		}
 	}()
 
-	jcfg := &configJSON{}
+	jcfg = &configJSON{}
 
 	// Private Key
 	pkeyBytes, err := cfg.PrivateKey.Bytes()
@@ -415,7 +431,6 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 	jcfg.DisableRepinning = cfg.DisableRepinning
 	jcfg.PeerstoreFile = cfg.PeerstoreFile
 
-	raw, err = json.MarshalIndent(jcfg, "", "    ")
 	return
 }
 

cluster_config_test.go

@@ -187,15 +187,6 @@ func TestLoadJSON(t *testing.T) {
 			t.Error("expected default replication factors")
 		}
 	})
-
-	t.Run("env var override", func(t *testing.T) {
-		os.Setenv("CLUSTER_PEERNAME", "envsetpeername")
-		cfg := &Config{}
-		cfg.LoadJSON(ccfgTestJSON)
-		if cfg.Peername != "envsetpeername" {
-			t.Fatal("failed to override peername with env var")
-		}
-	})
 }
 
 func TestToJSON(t *testing.T) {
@@ -220,6 +211,16 @@ func TestDefault(t *testing.T) {
 	}
 }
 
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_PEERNAME", "envsetpeername")
+	cfg := &Config{}
+	cfg.Default()
+	cfg.ApplyEnvVars()
+	if cfg.Peername != "envsetpeername" {
+		t.Fatal("failed to override peername with env var")
+	}
+}
+
 func TestValidate(t *testing.T) {
 	cfg := &Config{}
 	cfg.Default()

cmd/ipfs-cluster-service/daemon.go

@@ -69,7 +69,7 @@ func daemon(c *cli.Context) error {
 	// always wait for configuration to be saved
 	defer cfgMgr.Shutdown()
 
-	err = cfgMgr.LoadJSONFromFile(configPath)
+	err = cfgMgr.LoadJSONFileAndEnv(configPath)
 	checkErr("loading configuration", err)
 
 	if c.Bool("stats") {

cmd/ipfs-cluster-service/main.go

@@ -240,6 +240,9 @@ configuration.
 				err := cfgMgr.Default()
 				checkErr("generating default configuration", err)
 
+				err = cfgMgr.ApplyEnvVars()
+				checkErr("applying environment variables to configuration", err)
+
 				// Set user secret
 				if userSecretDefined {
 					cfgs.clusterCfg.Secret = userSecret
@@ -439,7 +442,7 @@ the mth data folder (m currently defaults to 5)
 						}
 
 						cfgMgr, cfgs := makeConfigs()
-						err = cfgMgr.LoadJSONFromFile(configPath)
+						err = cfgMgr.LoadJSONFileAndEnv(configPath)
 						checkErr("reading configuration", err)
 
 						err = cleanupState(cfgs.consensusCfg)
@@ -501,18 +504,14 @@ func setupDebug() {
 }
 
 func userProvidedSecret(enterSecret bool) ([]byte, bool) {
-	var secret string
 	if enterSecret {
-		secret = promptUser("Enter cluster secret (32-byte hex string): ")
+		secret := promptUser("Enter cluster secret (32-byte hex string): ")
-	} else if envSecret, envSecretDefined := os.LookupEnv("CLUSTER_SECRET"); envSecretDefined {
+		decodedSecret, err := ipfscluster.DecodeClusterSecret(secret)
-		secret = envSecret
+		checkErr("parsing user-provided secret", err)
-	} else {
+		return decodedSecret, true
-		return nil, false
 	}
 
-	decodedSecret, err := ipfscluster.DecodeClusterSecret(secret)
+	return nil, false
-	checkErr("parsing user-provided secret", err)
-	return decodedSecret, true
 }
 
 func promptUser(msg string) string {

cmd/ipfs-cluster-service/state.go

@@ -34,7 +34,7 @@ func upgrade(ctx context.Context) error {
 
 	cfgMgr, cfgs := makeConfigs()
 
-	err = cfgMgr.LoadJSONFromFile(configPath)
+	err = cfgMgr.LoadJSONFileAndEnv(configPath)
 	if err != nil {
 		return err
 	}
@@ -65,7 +65,7 @@ func restoreStateFromDisk(ctx context.Context) (*mapstate.MapState, bool, error)
 
 	cfgMgr, cfgs := makeConfigs()
 
-	err := cfgMgr.LoadJSONFromFile(configPath)
+	err := cfgMgr.LoadJSONFileAndEnv(configPath)
 	if err != nil {
 		return nil, false, err
 	}
@@ -108,7 +108,7 @@ func stateImport(ctx context.Context, r io.Reader) error {
 
 	cfgMgr, cfgs := makeConfigs()
 
-	err := cfgMgr.LoadJSONFromFile(configPath)
+	err := cfgMgr.LoadJSONFileAndEnv(configPath)
 	if err != nil {
 		return err
 	}

config/config.go

@@ -34,6 +34,8 @@ type ComponentConfig interface {
 	ToJSON() ([]byte, error)
 	// Sets default working values
 	Default() error
+	// Sets values from environment variables
+	ApplyEnvVars() error
 	// Allows this component to work under a subfolder
 	SetBaseDir(string)
 	// Checks that the configuration is valid
@@ -225,6 +227,30 @@ func (cfg *Manager) Default() error {
 	return nil
 }
 
+// ApplyEnvVars overrides configuration fields with any values found
+// in environment variables.
+func (cfg *Manager) ApplyEnvVars() error {
+	for _, section := range cfg.sections {
+		for k, compcfg := range section {
+			logger.Debugf("applying environment variables conf for %s", k)
+			err := compcfg.ApplyEnvVars()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if cfg.clusterConfig != nil {
+		logger.Debugf("applying environment variables conf for cluster")
+		err := cfg.clusterConfig.ApplyEnvVars()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // RegisterComponent lets the Manager load and save component configurations
 func (cfg *Manager) RegisterComponent(t SectionType, ccfg ComponentConfig) {
 	cfg.wg.Add(1)
@@ -296,6 +322,17 @@ func (cfg *Manager) LoadJSONFromFile(path string) error {
 	return err
 }
 
+// LoadJSONFileAndEnv calls LoadJSONFromFile followed by ApplyEnvVars,
+// reading and parsing a Configuration file and then overriding fields
+// with any values found in environment variables.
+func (cfg *Manager) LoadJSONFileAndEnv(path string) error {
+	if err := cfg.LoadJSONFromFile(path); err != nil {
+		return err
+	}
+
+	return cfg.ApplyEnvVars()
+}
+
 // LoadJSON parses configurations for all registered components,
 // In order to work, component configurations must have been registered
 // beforehand with RegisterComponent.

config/config_test.go

@@ -30,6 +30,10 @@ func (m *mockCfg) Default() error {
 	return nil
 }
 
+func (m *mockCfg) ApplyEnvVars() error {
+	return nil
+}
+
 func (m *mockCfg) Validate() error {
 	return nil
 }

consensus/raft/config.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/ipfs/ipfs-cluster/api"
 	"github.com/ipfs/ipfs-cluster/config"
+	"github.com/kelseyhightower/envconfig"
 
 	hraft "github.com/hashicorp/raft"
 	peer "github.com/libp2p/go-libp2p-peer"
@@ -17,6 +18,7 @@ import (
 // ConfigKey is the default configuration key for holding this component's
 // configuration section.
 var configKey = "raft"
+var envConfigKey = "cluster_raft"
 
 // Configuration defaults
 var (
@@ -185,6 +187,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	parseDuration := func(txt string) time.Duration {
 		d, _ := time.ParseDuration(txt)
 		if txt != "" && d == 0 {
@@ -230,7 +236,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON returns the pretty JSON representation of a Config.
 func (cfg *Config) ToJSON() ([]byte, error) {
-	jcfg := &jsonConfig{
+	jcfg := cfg.toJSONConfig()
+
+	return config.DefaultJSONMarshal(jcfg)
+}
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
 		DataFolder:           cfg.DataFolder,
 		InitPeerset:          api.PeersToStrings(cfg.InitPeerset),
 		WaitForLeaderTimeout: cfg.WaitForLeaderTimeout.String(),
@@ -247,8 +259,6 @@ func (cfg *Config) ToJSON() ([]byte, error) {
 		SnapshotThreshold:    cfg.RaftConfig.SnapshotThreshold,
 		LeaderLeaseTimeout:   cfg.RaftConfig.LeaderLeaseTimeout.String(),
 	}
-
-	return config.DefaultJSONMarshal(jcfg)
 }
 
 // Default initializes this configuration with working defaults.
@@ -272,6 +282,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // GetDataFolder returns the Raft data folder that we are using.
 func (cfg *Config) GetDataFolder() string {
 	if cfg.DataFolder == "" {

consensus/raft/config_test.go

@@ -2,6 +2,7 @@ package raft
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
 
 	hraft "github.com/hashicorp/raft"
@@ -106,3 +107,14 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_RAFT_COMMITRETRIES", "300")
+	cfg := &Config{}
+	cfg.Default()
+	cfg.ApplyEnvVars()
+
+	if cfg.CommitRetries != 300 {
+		t.Fatal("failed to override commit_retries with env var")
+	}
+}

informer/disk/config.go

@@ -6,9 +6,11 @@ import (
 	"time"
 
 	"github.com/ipfs/ipfs-cluster/config"
+	"github.com/kelseyhightower/envconfig"
 )
 
 const configKey = "disk"
+const envConfigKey = "cluster_disk"
 
 // Default values for disk Config
 const (
@@ -53,6 +55,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -76,6 +91,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 		return err
 	}
 
+	cfg.Default()
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	t, _ := time.ParseDuration(jcfg.MetricTTL)
 	cfg.MetricTTL = t
 
@@ -94,11 +115,15 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 // ToJSON generates a JSON-formatted human-friendly representation of this
 // Config.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
-	jcfg := &jsonConfig{}
+	jcfg := cfg.toJSONConfig()
-
-	jcfg.MetricTTL = cfg.MetricTTL.String()
-	jcfg.Type = cfg.Type.String()
 
 	raw, err = config.DefaultJSONMarshal(jcfg)
 	return
 }
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
+		MetricTTL: cfg.MetricTTL.String(),
+		Type:      cfg.Type.String(),
+	}
+}

informer/disk/config_test.go

@@ -2,7 +2,9 @@ package disk
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -81,3 +83,13 @@ func TestDefault(t *testing.T) {
 		t.Fatal("MetricRepoSize is a valid type")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_DISK_METRICTTL", "22s")
+	cfg := &Config{}
+	cfg.ApplyEnvVars()
+
+	if cfg.MetricTTL != 22*time.Second {
+		t.Fatal("failed to override metric_ttl with env var")
+	}
+}

informer/numpin/config.go

@@ -6,9 +6,11 @@ import (
 	"time"
 
 	"github.com/ipfs/ipfs-cluster/config"
+	"github.com/kelseyhightower/envconfig"
 )
 
 const configKey = "numpin"
+const envConfigKey = "cluster_numpin"
 
 // These are the default values for a Config.
 const (
@@ -38,6 +40,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this configuration have
 // sensible values.
 func (cfg *Config) Validate() error {
@@ -56,6 +71,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 		return err
 	}
 
+	cfg.Default()
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	t, _ := time.ParseDuration(jcfg.MetricTTL)
 	cfg.MetricTTL = t
 
@@ -64,9 +85,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() ([]byte, error) {
-	jcfg := &jsonConfig{}
+	jcfg := cfg.toJSONConfig()
-
-	jcfg.MetricTTL = cfg.MetricTTL.String()
 
 	return config.DefaultJSONMarshal(jcfg)
 }
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
+		MetricTTL: cfg.MetricTTL.String(),
+	}
+}

informer/numpin/config_test.go

@@ -2,7 +2,9 @@ package numpin
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -55,3 +57,13 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_NUMPIN_METRICTTL", "22s")
+	cfg := &Config{}
+	cfg.ApplyEnvVars()
+
+	if cfg.MetricTTL != 22*time.Second {
+		t.Fatal("failed to override metric_ttl with env var")
+	}
+}

ipfsconn/ipfshttp/config.go

@@ -6,12 +6,15 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/kelseyhightower/envconfig"
+
 	"github.com/ipfs/ipfs-cluster/config"
 
 	ma "github.com/multiformats/go-multiaddr"
 )
 
 const configKey = "ipfshttp"
+const envConfigKey = "cluster_ipfshttp"
 
 // Default values for Config.
 const (
@@ -89,6 +92,22 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return err
+	}
+
+	err = envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have sensible values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -133,6 +152,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	nodeAddr, err := ma.NewMultiaddr(jcfg.NodeMultiaddress)
 	if err != nil {
 		return fmt.Errorf("error parsing ipfs_node_multiaddress: %s", err)
@@ -158,6 +181,16 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() (raw []byte, err error) {
+	jcfg, err := cfg.toJSONConfig()
+	if err != nil {
+		return
+	}
+
+	raw, err = config.DefaultJSONMarshal(jcfg)
+	return
+}
+
+func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) {
 	// Multiaddress String() may panic
 	defer func() {
 		if r := recover(); r != nil {
@@ -165,7 +198,7 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 		}
 	}()
 
-	jcfg := &jsonConfig{}
+	jcfg = &jsonConfig{}
 
 	// Set all configuration fields
 	jcfg.NodeMultiaddress = cfg.NodeAddr.String()
@@ -175,6 +208,5 @@ func (cfg *Config) ToJSON() (raw []byte, err error) {
 	jcfg.PinTimeout = cfg.PinTimeout.String()
 	jcfg.UnpinTimeout = cfg.UnpinTimeout.String()
 
-	raw, err = config.DefaultJSONMarshal(jcfg)
 	return
 }

ipfsconn/ipfshttp/config_test.go

@@ -2,7 +2,9 @@ package ipfshttp
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -59,3 +61,14 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVar(t *testing.T) {
+	os.Setenv("CLUSTER_IPFSHTTP_PINTIMEOUT", "22m")
+	cfg := &Config{}
+	cfg.Default()
+	cfg.ApplyEnvVars()
+
+	if cfg.PinTimeout != 22*time.Minute {
+		t.Fatal("failed to override pin_timeout with env var")
+	}
+}

monitor/basic/config.go

@@ -6,9 +6,11 @@ import (
 	"time"
 
 	"github.com/ipfs/ipfs-cluster/config"
+	"github.com/kelseyhightower/envconfig"
 )
 
 const configKey = "monbasic"
+const envConfigKey = "cluster_monbasic"
 
 // Default values for this Config.
 const (
@@ -37,6 +39,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -56,6 +71,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 		return err
 	}
 
+	cfg.Default()
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	interval, _ := time.ParseDuration(jcfg.CheckInterval)
 	cfg.CheckInterval = interval
 
@@ -64,9 +85,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() ([]byte, error) {
-	jcfg := &jsonConfig{}
+	jcfg := cfg.toJSONConfig()
-
-	jcfg.CheckInterval = cfg.CheckInterval.String()
 
 	return json.MarshalIndent(jcfg, "", "    ")
 }
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
+		CheckInterval: cfg.CheckInterval.String(),
+	}
+}

monitor/basic/config_test.go

@@ -2,7 +2,9 @@ package basic
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -55,3 +57,13 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_MONBASIC_CHECKINTERVAL", "22s")
+	cfg := &Config{}
+	cfg.ApplyEnvVars()
+
+	if cfg.CheckInterval != 22*time.Second {
+		t.Fatal("failed to override check_interval with env var")
+	}
+}

monitor/pubsubmon/config.go

@@ -6,9 +6,11 @@ import (
 	"time"
 
 	"github.com/ipfs/ipfs-cluster/config"
+	"github.com/kelseyhightower/envconfig"
 )
 
 const configKey = "pubsubmon"
+const envConfigKey = "cluster_pubsubmon"
 
 // Default values for this Config.
 const (
@@ -37,6 +39,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -56,6 +71,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 		return err
 	}
 
+	cfg.Default()
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	interval, _ := time.ParseDuration(jcfg.CheckInterval)
 	cfg.CheckInterval = interval
 
@@ -64,9 +85,13 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() ([]byte, error) {
-	jcfg := &jsonConfig{}
+	jcfg := cfg.toJSONConfig()
-
-	jcfg.CheckInterval = cfg.CheckInterval.String()
 
 	return json.MarshalIndent(jcfg, "", "    ")
 }
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
+		CheckInterval: cfg.CheckInterval.String(),
+	}
+}

monitor/pubsubmon/config_test.go

@@ -2,7 +2,9 @@ package pubsubmon
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
+	"time"
 )
 
 var cfgJSON = []byte(`
@@ -55,3 +57,13 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_PUBSUBMON_CHECKINTERVAL", "22s")
+	cfg := &Config{}
+	cfg.ApplyEnvVars()
+
+	if cfg.CheckInterval != 22*time.Second {
+		t.Fatal("failed to override check_interval with env var")
+	}
+}

observations/config.go

@@ -14,7 +14,8 @@ import (
 
 const metricsConfigKey = "metrics"
 const tracingConfigKey = "tracing"
-const envConfigKey = "cluster_observations"
+const metricsEnvConfigKey = "cluster_metrics"
+const tracingEnvConfigKey = "cluster_tracing"
 
 // Default values for this Config.
 const (
@@ -58,6 +59,19 @@ func (cfg *MetricsConfig) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *MetricsConfig) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(metricsEnvConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *MetricsConfig) Validate() error {
@@ -84,13 +98,11 @@ func (cfg *MetricsConfig) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
-	// override json config with env var
+	return cfg.applyJSONConfig(jcfg)
-	err = envconfig.Process(envConfigKey, jcfg)
+}
-	if err != nil {
-		return err
-	}
 
-	err = cfg.loadMetricsOptions(jcfg)
+func (cfg *MetricsConfig) applyJSONConfig(jcfg *jsonMetricsConfig) error {
+	err := cfg.loadMetricsOptions(jcfg)
 	if err != nil {
 		return err
 	}
@@ -118,13 +130,17 @@ func (cfg *MetricsConfig) loadMetricsOptions(jcfg *jsonMetricsConfig) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *MetricsConfig) ToJSON() ([]byte, error) {
-	jcfg := &jsonMetricsConfig{
+	jcfg := cfg.toJSONConfig()
+
+	return config.DefaultJSONMarshal(jcfg)
+}
+
+func (cfg *MetricsConfig) toJSONConfig() *jsonMetricsConfig {
+	return &jsonMetricsConfig{
 		EnableStats:        cfg.EnableStats,
 		PrometheusEndpoint: cfg.PrometheusEndpoint.String(),
 		ReportingInterval:  cfg.ReportingInterval.String(),
 	}
-
-	return config.DefaultJSONMarshal(jcfg)
 }
 
 // TracingConfig configures tracing.
@@ -159,6 +175,19 @@ func (cfg *TracingConfig) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *TracingConfig) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(tracingEnvConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *TracingConfig) Validate() error {
@@ -185,13 +214,11 @@ func (cfg *TracingConfig) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
-	// override json config with env var
+	return cfg.applyJSONConfig(jcfg)
-	err = envconfig.Process(envConfigKey, jcfg)
+}
-	if err != nil {
-		return err
-	}
 
-	err = cfg.loadTracingOptions(jcfg)
+func (cfg *TracingConfig) applyJSONConfig(jcfg *jsonTracingConfig) error {
+	err := cfg.loadTracingOptions(jcfg)
 	if err != nil {
 		return err
 	}
@@ -214,12 +241,16 @@ func (cfg *TracingConfig) loadTracingOptions(jcfg *jsonTracingConfig) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *TracingConfig) ToJSON() ([]byte, error) {
-	jcfg := &jsonTracingConfig{
+	jcfg := cfg.toJSONConfig()
+
+	return config.DefaultJSONMarshal(jcfg)
+}
+
+func (cfg *TracingConfig) toJSONConfig() *jsonTracingConfig {
+	return &jsonTracingConfig{
 		EnableTracing:       cfg.EnableTracing,
 		JaegerAgentEndpoint: cfg.JaegerAgentEndpoint.String(),
 		SamplingProb:        cfg.SamplingProb,
 		ServiceName:         cfg.ServiceName,
 	}
-
-	return config.DefaultJSONMarshal(jcfg)
 }

observations/config_test.go

@@ -0,0 +1,26 @@
+package observations
+
+import (
+	"os"
+	"testing"
+)
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_METRICS_ENABLESTATS", "true")
+	mcfg := &MetricsConfig{}
+	mcfg.Default()
+	mcfg.ApplyEnvVars()
+
+	if !mcfg.EnableStats {
+		t.Fatal("failed to override enable_stats with env var")
+	}
+
+	os.Setenv("CLUSTER_TRACING_ENABLETRACING", "true")
+	tcfg := &TracingConfig{}
+	tcfg.Default()
+	tcfg.ApplyEnvVars()
+
+	if !tcfg.EnableTracing {
+		t.Fatal("failed to override enable_tracing with env var")
+	}
+}

pintracker/maptracker/config.go

@@ -4,10 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 
+	"github.com/kelseyhightower/envconfig"
+
 	"github.com/ipfs/ipfs-cluster/config"
 )
 
 const configKey = "maptracker"
+const envConfigKey = "cluster_maptracker"
 
 // Default values for this Config.
 const (
@@ -44,6 +47,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -69,6 +85,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	config.SetIfNotDefault(jcfg.MaxPinQueueSize, &cfg.MaxPinQueueSize)
 	config.SetIfNotDefault(jcfg.ConcurrentPins, &cfg.ConcurrentPins)
 
@@ -77,10 +97,14 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() ([]byte, error) {
-	jcfg := &jsonConfig{}
+	jcfg := cfg.toJSONConfig()
-
-	jcfg.MaxPinQueueSize = cfg.MaxPinQueueSize
-	jcfg.ConcurrentPins = cfg.ConcurrentPins
 
 	return config.DefaultJSONMarshal(jcfg)
 }
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
+		MaxPinQueueSize: cfg.MaxPinQueueSize,
+		ConcurrentPins:  cfg.ConcurrentPins,
+	}
+}

pintracker/maptracker/config_test.go

@@ -2,6 +2,7 @@ package maptracker
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
 )
 
@@ -59,3 +60,13 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_MAPTRACKER_CONCURRENTPINS", "22")
+	cfg := &Config{}
+	cfg.ApplyEnvVars()
+
+	if cfg.ConcurrentPins != 22 {
+		t.Fatal("failed to override concurrent_pins with env var")
+	}
+}

pintracker/stateless/config.go

@@ -4,10 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 
+	"github.com/kelseyhightower/envconfig"
+
 	"github.com/ipfs/ipfs-cluster/config"
 )
 
 const configKey = "stateless"
+const envConfigKey = "cluster_stateless"
 
 // Default values for this Config.
 const (
@@ -44,6 +47,19 @@ func (cfg *Config) Default() error {
 	return nil
 }
 
+// ApplyEnvVars fills in any Config fields found
+// as environment variables.
+func (cfg *Config) ApplyEnvVars() error {
+	jcfg := cfg.toJSONConfig()
+
+	err := envconfig.Process(envConfigKey, jcfg)
+	if err != nil {
+		return err
+	}
+
+	return cfg.applyJSONConfig(jcfg)
+}
+
 // Validate checks that the fields of this Config have working values,
 // at least in appearance.
 func (cfg *Config) Validate() error {
@@ -69,6 +85,10 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 	cfg.Default()
 
+	return cfg.applyJSONConfig(jcfg)
+}
+
+func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	config.SetIfNotDefault(jcfg.MaxPinQueueSize, &cfg.MaxPinQueueSize)
 	config.SetIfNotDefault(jcfg.ConcurrentPins, &cfg.ConcurrentPins)
 
@@ -77,10 +97,14 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 // ToJSON generates a human-friendly JSON representation of this Config.
 func (cfg *Config) ToJSON() ([]byte, error) {
-	jcfg := &jsonConfig{}
+	jcfg := cfg.toJSONConfig()
-
-	jcfg.MaxPinQueueSize = cfg.MaxPinQueueSize
-	jcfg.ConcurrentPins = cfg.ConcurrentPins
 
 	return config.DefaultJSONMarshal(jcfg)
 }
+
+func (cfg *Config) toJSONConfig() *jsonConfig {
+	return &jsonConfig{
+		MaxPinQueueSize: cfg.MaxPinQueueSize,
+		ConcurrentPins:  cfg.ConcurrentPins,
+	}
+}

pintracker/stateless/config_test.go

@@ -2,6 +2,7 @@ package stateless
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
 )
 
@@ -59,3 +60,13 @@ func TestDefault(t *testing.T) {
 		t.Fatal("expected error validating")
 	}
 }
+
+func TestApplyEnvVars(t *testing.T) {
+	os.Setenv("CLUSTER_STATELESS_CONCURRENTPINS", "22")
+	cfg := &Config{}
+	cfg.ApplyEnvVars()
+
+	if cfg.ConcurrentPins != 22 {
+		t.Fatal("failed to override concurrent_pins with env var")
+	}
+}