Skip to content

Commit 60e4e45

Browse files
author
Jason Yellick
committed
Add orderer config mechanism
This changeset adds an orderer.yaml config file which specifies the defaults which were previously hardcoded. It also correspondingly removes the hardcoded defaults and utilizes the configuration in those places. It resolves https://jira.hyperledger.org/browse/FAB-386 This configuration is being done using a variant of Viper's Unmarshal. In newer versions of Viper there is a new function UnmarshalExact which throws errors when Unmarshaling encounters config fields which are unexpected. However, there are two outstanding bugs around this Viper feature which affect us and do not make pulling the newer code worthwhile at this point. 1. Unmarshaling does not appropriately support time durations spf13/viper#105 spf13/viper#205 2. Unmarshaling does not correctly handle env overrides of nested config parameters spf13/viper#160 This changeset includes a stand-in implementation of UnmarshalExact which does not suffer from these Viper bugs. These workarounds should be removed once fixes are pushed upstream. Committing to reclaim this commit after a little Gerrit accident Change-Id: I931b955b0d8fdaacb240a1b480eb695109774e38 Signed-off-by: Jason Yellick <jyellick@us.ibm.com>
1 parent eefbf7c commit 60e4e45

File tree

11 files changed

+423
-44
lines changed

11 files changed

+423
-44
lines changed

‎bddtests/docker-compose-orderer-solo.yml

+8-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
orderer0:
22
image: hyperledger/fabric-orderer
33
environment:
4-
- ORDERER_LISTEN_ADDRESS=0.0.0.0
5-
- ORDERER_LISTEN_PORT=5005
6-
#- ORDERER_WINDOW_SIZE_MAX=1000 # TODO (implement)
7-
#- ORDERER_BATCH_TIMEOUT=10s # TODO (implement)
8-
#- ORDERER_BATCH_SIZE=10 # TODO (implement)
9-
#- ORDERER_BLOCK_HISTORY_SIZE=100 # TODO (implement)
4+
- ORDERER_GENERAL_ORDERERTYPE=solo
5+
- ORDERER_GENERAL_LEDGERTYPE=ram
6+
- ORDERER_GENERAL_BATCHTIMEOUT=10s
7+
- ORDERER_GENERAL_BATCHSIZE=10
8+
- ORDERER_GENERAL_MAXWINDOWSIZE=1000
9+
- ORDERER_GENERAL_LISTENADDRESS=0.0.0.0
10+
- ORDERER_GENERAL_LISTENPORT=5005
11+
- ORDERER_RAMLEDGER_HISTORY_SIZE=100
1012

1113
working_dir: /opt/gopath/src/github.com/hyperledger/fabric/orderer
1214
command: orderer

‎orderer/config/config.go

+167
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package config
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"path/filepath"
23+
"strings"
24+
"time"
25+
26+
"github.com/op/go-logging"
27+
"github.com/spf13/viper"
28+
)
29+
30+
var logger = logging.MustGetLogger("orderer/config")
31+
32+
func init() {
33+
logging.SetLevel(logging.DEBUG, "")
34+
}
35+
36+
// Prefix is the default config prefix for the orderer
37+
const Prefix string = "ORDERER"
38+
39+
// General contains config which should be common among all orderer types
40+
type General struct {
41+
OrdererType string
42+
LedgerType string
43+
BatchTimeout time.Duration
44+
BatchSize uint
45+
QueueSize uint
46+
MaxWindowSize uint
47+
ListenAddress string
48+
ListenPort uint16
49+
}
50+
51+
// RAMLedger contains config for the RAM ledger
52+
type RAMLedger struct {
53+
HistorySize uint
54+
}
55+
56+
// FileLedger contains config for the File ledger
57+
type FileLedger struct {
58+
Location string
59+
Prefix string
60+
}
61+
62+
// TopLevel directly corresponds to the orderer config yaml
63+
// Note, for non 1-1 mappings, you may append
64+
// something like `mapstructure:"weirdFoRMat"` to
65+
// modify the default mapping, see the "Unmarshal"
66+
// section of https://github.com/spf13/viper for more info
67+
type TopLevel struct {
68+
General General
69+
RAMLedger RAMLedger
70+
FileLedger FileLedger
71+
}
72+
73+
var defaults = TopLevel{
74+
General: General{
75+
OrdererType: "solo",
76+
LedgerType: "ram",
77+
BatchTimeout: 10 * time.Second,
78+
BatchSize: 10,
79+
QueueSize: 1000,
80+
MaxWindowSize: 1000,
81+
ListenAddress: "127.0.0.1",
82+
ListenPort: 5151,
83+
},
84+
RAMLedger: RAMLedger{
85+
HistorySize: 10000,
86+
},
87+
FileLedger: FileLedger{
88+
Location: "",
89+
Prefix: "hyperledger-fabric-rawledger",
90+
},
91+
}
92+
93+
func (c *TopLevel) completeInitialization() {
94+
defer logger.Infof("Validated configuration to: %+v", c)
95+
96+
for {
97+
switch {
98+
case c.General.OrdererType == "":
99+
logger.Infof("General.OrdererType unset, setting to %s", defaults.General.OrdererType)
100+
c.General.OrdererType = defaults.General.OrdererType
101+
case c.General.LedgerType == "":
102+
logger.Infof("General.LedgerType unset, setting to %s", defaults.General.LedgerType)
103+
c.General.LedgerType = defaults.General.LedgerType
104+
case c.General.BatchTimeout == 0:
105+
logger.Infof("General.BatchTimeout unset, setting to %s", defaults.General.BatchTimeout)
106+
c.General.BatchTimeout = defaults.General.BatchTimeout
107+
case c.General.BatchSize == 0:
108+
logger.Infof("General.BatchSize unset, setting to %s", defaults.General.BatchSize)
109+
c.General.BatchSize = defaults.General.BatchSize
110+
case c.General.QueueSize == 0:
111+
logger.Infof("General.QueueSize unset, setting to %s", defaults.General.QueueSize)
112+
c.General.QueueSize = defaults.General.QueueSize
113+
case c.General.MaxWindowSize == 0:
114+
logger.Infof("General.MaxWindowSize unset, setting to %s", defaults.General.MaxWindowSize)
115+
c.General.MaxWindowSize = defaults.General.MaxWindowSize
116+
case c.General.ListenAddress == "":
117+
logger.Infof("General.ListenAddress unset, setting to %s", defaults.General.ListenAddress)
118+
c.General.ListenAddress = defaults.General.ListenAddress
119+
case c.General.ListenPort == 0:
120+
logger.Infof("General.ListenPort unset, setting to %s", defaults.General.ListenPort)
121+
c.General.ListenPort = defaults.General.ListenPort
122+
case c.FileLedger.Prefix == "":
123+
logger.Infof("FileLedger.Prefix unset, setting to %s", defaults.FileLedger.Prefix)
124+
c.FileLedger.Prefix = defaults.FileLedger.Prefix
125+
default:
126+
return
127+
}
128+
}
129+
}
130+
131+
// Load parses the orderer.yaml file and environment, producing a struct suitable for config use
132+
func Load() *TopLevel {
133+
config := viper.New()
134+
135+
// for environment variables
136+
config.SetEnvPrefix(Prefix)
137+
config.AutomaticEnv()
138+
replacer := strings.NewReplacer(".", "_")
139+
config.SetEnvKeyReplacer(replacer)
140+
141+
config.SetConfigName("orderer")
142+
config.AddConfigPath("./")
143+
config.AddConfigPath("../orderer/")
144+
config.AddConfigPath("../../orderer/")
145+
// Path to look for the config file in based on GOPATH
146+
gopath := os.Getenv("GOPATH")
147+
for _, p := range filepath.SplitList(gopath) {
148+
ordererPath := filepath.Join(p, "src/github.com/hyperledger/fabric/orderer/")
149+
config.AddConfigPath(ordererPath)
150+
}
151+
152+
err := config.ReadInConfig()
153+
if err != nil {
154+
panic(fmt.Errorf("Error reading %s plugin config: %s", Prefix, err))
155+
}
156+
157+
var uconf TopLevel
158+
159+
err = ExactWithDateUnmarshal(config, &uconf)
160+
if err != nil {
161+
panic(fmt.Errorf("Error unmarshaling into structure: %s", err))
162+
}
163+
164+
uconf.completeInitialization()
165+
166+
return &uconf
167+
}

‎orderer/config/config_test.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package config
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"testing"
23+
24+
"github.com/spf13/viper"
25+
)
26+
27+
func TestGoodConfig(t *testing.T) {
28+
config := Load()
29+
if config == nil {
30+
t.Fatalf("Could not load config")
31+
}
32+
t.Logf("%+v", config)
33+
}
34+
35+
func TestBadConfig(t *testing.T) {
36+
config := viper.New()
37+
config.SetConfigName("orderer")
38+
config.AddConfigPath("../")
39+
40+
err := config.ReadInConfig()
41+
if err != nil {
42+
t.Fatalf("Error reading %s plugin config: %s", Prefix, err)
43+
}
44+
45+
var uconf struct{}
46+
47+
err = ExactWithDateUnmarshal(config, &uconf)
48+
if err == nil {
49+
t.Fatalf("Should have failed to unmarshal")
50+
}
51+
}
52+
53+
// TestEnvInnerVar verifies that with the Unmarshal function that
54+
// the environmental overrides still work on internal vars. This was
55+
// a bug in the original viper implementation that is worked around in
56+
// the Load codepath for now
57+
func TestEnvInnerVar(t *testing.T) {
58+
envVar := "ORDERER_GENERAL_LISTENPORT"
59+
envVal := uint16(80)
60+
os.Setenv(envVar, fmt.Sprintf("%d", envVal))
61+
defer os.Unsetenv(envVar)
62+
config := Load()
63+
64+
if config == nil {
65+
t.Fatalf("Could not load config")
66+
}
67+
68+
if config.General.ListenPort != envVal {
69+
t.Fatalf("Environmental override of inner config did not work")
70+
}
71+
}

‎orderer/config/config_util.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package config
18+
19+
import (
20+
"github.com/mitchellh/mapstructure"
21+
"github.com/spf13/viper"
22+
)
23+
24+
func getKeysRecursively(base string, v *viper.Viper, nodeKeys map[string]interface{}) map[string]interface{} {
25+
result := make(map[string]interface{})
26+
for key := range nodeKeys {
27+
fqKey := base + key
28+
val := v.Get(fqKey)
29+
if m, ok := val.(map[interface{}]interface{}); ok {
30+
logger.Debugf("Found map value for %s", fqKey)
31+
tmp := make(map[string]interface{})
32+
for ik, iv := range m {
33+
cik, ok := ik.(string)
34+
if !ok {
35+
panic("Non string key-entry")
36+
}
37+
tmp[cik] = iv
38+
}
39+
result[key] = getKeysRecursively(fqKey+".", v, tmp)
40+
} else {
41+
logger.Debugf("Found real value for %s setting to %T %v", fqKey, val, val)
42+
result[key] = val
43+
}
44+
}
45+
return result
46+
}
47+
48+
// ExactWithDateUnmarshal is intended to unmarshal a config file into a structure
49+
// producing error when extraneous variables are introduced and supporting
50+
// the time.Duration type
51+
func ExactWithDateUnmarshal(v *viper.Viper, output interface{}) error {
52+
baseKeys := v.AllSettings() // AllKeys doesn't actually return all keys, it only returns the base ones
53+
leafKeys := getKeysRecursively("", v, baseKeys)
54+
55+
logger.Infof("%+v", leafKeys)
56+
config := &mapstructure.DecoderConfig{
57+
ErrorUnused: true,
58+
Metadata: nil,
59+
Result: output,
60+
WeaklyTypedInput: true,
61+
DecodeHook: mapstructure.StringToTimeDurationHookFunc(),
62+
}
63+
64+
decoder, err := mapstructure.NewDecoder(config)
65+
if err != nil {
66+
return err
67+
}
68+
return decoder.Decode(leafKeys)
69+
}

0 commit comments

Comments
 (0)