From ae8e74453b62b77d4be4e26e7ce8c60f8574c262 Mon Sep 17 00:00:00 2001 From: Kishan Mohanbhai Sagathiya Date: Wed, 13 Nov 2019 18:35:25 +0530 Subject: [PATCH] Fix #937: Print full working configuration at startup Only when using debug mode Co-authored-by: Hector Sanjuan --- api/ipfsproxy/config.go | 10 +++++ api/rest/config.go | 14 ++++++- cluster_config.go | 13 +++++- cmd/ipfs-cluster-service/daemon.go | 5 ++- config/config.go | 53 +++++++++++++++++++++++-- config/config_test.go | 34 ++++++++++++++++ config/util.go | 63 ++++++++++++++++++++++++++++++ consensus/crdt/config.go | 5 +++ consensus/raft/config.go | 5 +++ datastore/badger/config.go | 5 +++ informer/disk/config.go | 5 +++ informer/numpin/config.go | 5 +++ ipfsconn/ipfshttp/config.go | 10 +++++ monitor/pubsubmon/config.go | 5 +++ observations/config.go | 10 +++++ pintracker/stateless/config.go | 5 +++ 16 files changed, 239 insertions(+), 8 deletions(-) diff --git a/api/ipfsproxy/config.go b/api/ipfsproxy/config.go index 3c002eac..f0faf763 100644 --- a/api/ipfsproxy/config.go +++ b/api/ipfsproxy/config.go @@ -333,3 +333,13 @@ func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) { return } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + jcfg, err := cfg.toJSONConfig() + if err != nil { + return nil, err + } + + return config.DisplayJSON(jcfg) +} diff --git a/api/rest/config.go b/api/rest/config.go index c506b5d7..8b877341 100644 --- a/api/rest/config.go +++ b/api/rest/config.go @@ -147,9 +147,9 @@ type jsonConfig struct { Libp2pListenMultiaddress ipfsconfig.Strings `json:"libp2p_listen_multiaddress,omitempty"` ID string `json:"id,omitempty"` - PrivateKey string `json:"private_key,omitempty"` + PrivateKey string `json:"private_key,omitempty" hidden:"true"` - BasicAuthCredentials map[string]string `json:"basic_auth_credentials"` + BasicAuthCredentials map[string]string `json:"basic_auth_credentials" hidden:"true"` HTTPLogFile string `json:"http_log_file"` Headers map[string][]string `json:"headers"` @@ -503,6 +503,16 @@ func (cfg *Config) corsOptions() *cors.Options { } } +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + jcfg, err := cfg.toJSONConfig() + if err != nil { + return nil, err + } + + return config.DisplayJSON(jcfg) +} + func newTLSConfig(certFile, keyFile string) (*tls.Config, error) { cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { diff --git a/cluster_config.go b/cluster_config.go index 5d02add3..d29f8d5d 100644 --- a/cluster_config.go +++ b/cluster_config.go @@ -162,8 +162,8 @@ type Config struct { type configJSON struct { ID string `json:"id,omitempty"` Peername string `json:"peername"` - PrivateKey string `json:"private_key,omitempty"` - Secret string `json:"secret"` + PrivateKey string `json:"private_key,omitempty" hidden:"true"` + Secret string `json:"secret" hidden:"true"` LeaveOnShutdown bool `json:"leave_on_shutdown"` ListenMultiaddress ipfsconfig.Strings `json:"listen_multiaddress"` EnableRelayHop bool `json:"enable_relay_hop"` @@ -526,6 +526,15 @@ func (cfg *Config) GetPeerstorePath() string { return filepath.Join(cfg.BaseDir, filename) } +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + jcfg, err := cfg.toConfigJSON() + if err != nil { + return nil, err + } + return config.DisplayJSON(jcfg) +} + // DecodeClusterSecret parses a hex-encoded string, checks that it is exactly // 32 bytes long and returns its value as a byte-slice.x func DecodeClusterSecret(hexSecret string) ([]byte, error) { diff --git a/cmd/ipfs-cluster-service/daemon.go b/cmd/ipfs-cluster-service/daemon.go index 275712cd..da01340c 100644 --- a/cmd/ipfs-cluster-service/daemon.go +++ b/cmd/ipfs-cluster-service/daemon.go @@ -121,8 +121,11 @@ func createCluster( cfgs := cfgHelper.Configs() cfgMgr := cfgHelper.Manager() + cfgBytes, err := cfgMgr.ToDisplayJSON() + checkErr("getting configuration string", err) + logger.Debugf("Configuration:\n%s\n", cfgBytes) - ctx, err := tag.New(ctx, tag.Upsert(observations.HostKey, host.ID().Pretty())) + ctx, err = tag.New(ctx, tag.Upsert(observations.HostKey, host.ID().Pretty())) checkErr("tag context with host id", err) var apis []ipfscluster.API diff --git a/config/config.go b/config/config.go index bcbaa26f..91aba0fe 100644 --- a/config/config.go +++ b/config/config.go @@ -58,6 +58,8 @@ type ComponentConfig interface { // Provides a channel to signal the Manager that the configuration // should be persisted. SaveCh() <-chan struct{} + // ToDisplayJSON returns a string representing the config excluding hidden fields. + ToDisplayJSON() ([]byte, error) } // These are the component configuration types @@ -512,7 +514,6 @@ func (cfg *Manager) ToJSON() ([]byte, error) { if cfg.clusterConfig != nil { cfg.clusterConfig.SetBaseDir(dir) raw, err := cfg.clusterConfig.ToJSON() - if err != nil { return nil, err } @@ -541,6 +542,52 @@ func (cfg *Manager) ToJSON() ([]byte, error) { return nil } + err = cfg.applyUpdateJSONConfigs(jcfg, updateJSONConfigs) + if err != nil { + return nil, err + } + + return DefaultJSONMarshal(jcfg) +} + +// ToDisplayJSON returns a printable cluster configuration. +func (cfg *Manager) ToDisplayJSON() ([]byte, error) { + jcfg := &jsonConfig{} + + if cfg.clusterConfig != nil { + raw, err := cfg.clusterConfig.ToDisplayJSON() + if err != nil { + return nil, err + } + jcfg.Cluster = new(json.RawMessage) + *jcfg.Cluster = raw + } + + updateJSONConfigs := func(section Section, dest *jsonSection) error { + for k, v := range section { + j, err := v.ToDisplayJSON() + if err != nil { + return err + } + if *dest == nil { + *dest = make(jsonSection) + } + jsonSection := *dest + jsonSection[k] = new(json.RawMessage) + *jsonSection[k] = j + } + return nil + } + + err := cfg.applyUpdateJSONConfigs(jcfg, updateJSONConfigs) + if err != nil { + return nil, err + } + + return DefaultJSONMarshal(jcfg) +} + +func (cfg *Manager) applyUpdateJSONConfigs(jcfg *jsonConfig, updateJSONConfigs func(section Section, dest *jsonSection) error) error { for _, t := range SectionTypes() { if t == Cluster { continue @@ -548,11 +595,11 @@ func (cfg *Manager) ToJSON() ([]byte, error) { jsection := jcfg.getSection(t) err := updateJSONConfigs(cfg.sections[t], jsection) if err != nil { - return nil, err + return err } } - return DefaultJSONMarshal(jcfg) + return nil } // IsLoadedFromJSON tells whether the given component belonging to diff --git a/config/config_test.go b/config/config_test.go index 74f962d7..6a456a46 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -92,6 +92,14 @@ func (m *mockCfg) Validate() error { return nil } +func (m *mockCfg) ToDisplayJSON() ([]byte, error) { + return []byte(` + { + "a":"b" + } + `), nil +} + func setupConfigManager() *Manager { cfg := NewManager() mockCfg := &mockCfg{} @@ -176,3 +184,29 @@ func TestSaveWithSource(t *testing.T) { t.Error("should have generated a source-only json") } } + +func TestDefaultJSONMarshalWithoutHiddenFields(t *testing.T) { + type s struct { + A string `json:"a_key"` + B string `json:"b_key" hidden:"true"` + } + cfg := s{ + A: "hi", + B: "there", + } + + expected := `{ + "a_key": "hi", + "b_key": "XXX_hidden_XXX" +}` + + res, err := DisplayJSON(&cfg) + if err != nil { + t.Fatal(err) + } + + if string(res) != expected { + t.Error("result does not match expected") + t.Error(string(res)) + } +} diff --git a/config/util.go b/config/util.go index ebda265a..91a1994a 100644 --- a/config/util.go +++ b/config/util.go @@ -3,6 +3,8 @@ package config import ( "encoding/json" "fmt" + "reflect" + "strings" "time" ) @@ -114,3 +116,64 @@ func ParseDurations(component string, args ...*DurationOpt) error { } return nil } + +type hiddenField struct{} + +func (hf hiddenField) MarshalJSON() ([]byte, error) { + return []byte(`"XXX_hidden_XXX"`), nil +} +func (hf hiddenField) UnmarshalJSON(b []byte) error { return nil } + +// DisplayJSON takes pointer to a JSON-friendly configuration struct and +// returns the JSON-encoded representation of it filtering out any struct +// fields marked with the tag `hidden:"true"`, but keeping fields marked +// with `"json:omitempty"`. +func DisplayJSON(cfg interface{}) ([]byte, error) { + cfg = reflect.Indirect(reflect.ValueOf(cfg)).Interface() + origStructT := reflect.TypeOf(cfg) + if origStructT.Kind() != reflect.Struct { + panic("the given argument should be a struct") + } + + hiddenFieldT := reflect.TypeOf(hiddenField{}) + + // create a new struct type with same fields + // but setting hidden fields as hidden. + finalStructFields := []reflect.StructField{} + for i := 0; i < origStructT.NumField(); i++ { + f := origStructT.Field(i) + hidden := f.Tag.Get("hidden") == "true" + if f.PkgPath != "" { // skip unexported + continue + } + if hidden { + f.Type = hiddenFieldT + } + + // remove omitempty from tag, ignore other tags except json + var jsonTags []string + for _, s := range strings.Split(f.Tag.Get("json"), ",") { + if s != "omitempty" { + jsonTags = append(jsonTags, s) + } + } + f.Tag = reflect.StructTag(fmt.Sprintf("json:\"%s\"", strings.Join(jsonTags, ","))) + + finalStructFields = append(finalStructFields, f) + } + + // Parse the original JSON into the new + // struct and re-convert it to JSON. + finalStructT := reflect.StructOf(finalStructFields) + finalValue := reflect.New(finalStructT) + data := finalValue.Interface() + origJSON, err := json.Marshal(cfg) + if err != nil { + return nil, err + } + err = json.Unmarshal(origJSON, data) + if err != nil { + return nil, err + } + return DefaultJSONMarshal(data) +} diff --git a/consensus/crdt/config.go b/consensus/crdt/config.go index 3f540a56..13be5423 100644 --- a/consensus/crdt/config.go +++ b/consensus/crdt/config.go @@ -191,3 +191,8 @@ func (cfg *Config) ApplyEnvVars() error { return cfg.applyJSONConfig(jcfg) } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/consensus/raft/config.go b/consensus/raft/config.go index 87be865f..6db8bb76 100644 --- a/consensus/raft/config.go +++ b/consensus/raft/config.go @@ -313,3 +313,8 @@ func (cfg *Config) GetDataFolder() string { } return cfg.DataFolder } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/datastore/badger/config.go b/datastore/badger/config.go index 741f24e0..1188a1da 100644 --- a/datastore/badger/config.go +++ b/datastore/badger/config.go @@ -234,3 +234,8 @@ func (cfg *Config) GetFolder() string { return filepath.Join(cfg.BaseDir, cfg.Folder) } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/informer/disk/config.go b/informer/disk/config.go index 7a3ee84f..e7cec8b2 100644 --- a/informer/disk/config.go +++ b/informer/disk/config.go @@ -127,3 +127,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig { MetricType: cfg.MetricType.String(), } } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/informer/numpin/config.go b/informer/numpin/config.go index b0a99f4b..83e8f468 100644 --- a/informer/numpin/config.go +++ b/informer/numpin/config.go @@ -95,3 +95,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig { MetricTTL: cfg.MetricTTL.String(), } } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/ipfsconn/ipfshttp/config.go b/ipfsconn/ipfshttp/config.go index 80da0e7a..a27c9979 100644 --- a/ipfsconn/ipfshttp/config.go +++ b/ipfsconn/ipfshttp/config.go @@ -205,3 +205,13 @@ func (cfg *Config) toJSONConfig() (jcfg *jsonConfig, err error) { return } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + jcfg, err := cfg.toJSONConfig() + if err != nil { + return nil, err + } + + return config.DisplayJSON(jcfg) +} diff --git a/monitor/pubsubmon/config.go b/monitor/pubsubmon/config.go index a6f48f7d..799d3091 100644 --- a/monitor/pubsubmon/config.go +++ b/monitor/pubsubmon/config.go @@ -111,3 +111,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig { FailureThreshold: &cfg.FailureThreshold, } } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/observations/config.go b/observations/config.go index 099aa557..3dbef357 100644 --- a/observations/config.go +++ b/observations/config.go @@ -144,6 +144,11 @@ func (cfg *MetricsConfig) toJSONConfig() *jsonMetricsConfig { } } +// ToDisplayJSON returns JSON config as a string. +func (cfg *MetricsConfig) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} + // TracingConfig configures tracing. type TracingConfig struct { config.Saver @@ -257,3 +262,8 @@ func (cfg *TracingConfig) toJSONConfig() *jsonTracingConfig { ServiceName: cfg.ServiceName, } } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *TracingConfig) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +} diff --git a/pintracker/stateless/config.go b/pintracker/stateless/config.go index 542506f7..790c6029 100644 --- a/pintracker/stateless/config.go +++ b/pintracker/stateless/config.go @@ -112,3 +112,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig { return jCfg } + +// ToDisplayJSON returns JSON config as a string. +func (cfg *Config) ToDisplayJSON() ([]byte, error) { + return config.DisplayJSON(cfg.toJSONConfig()) +}