Skip to content

Commit e43a11d

Browse files
authored
Merge hardcoded/default configuration with OTEL config file (#2211)
* Merge hardcoded/default configuration with OTEL config file Signed-off-by: Pavol Loffay <ploffay@redhat.com> * cleanup Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Copy the func for now Signed-off-by: Pavol Loffay <ploffay@redhat.com> * disable c* exporter Signed-off-by: Pavol Loffay <ploffay@redhat.com> * disable grpc Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Try depends on Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Use OTEL with wait for ready Signed-off-by: Pavol Loffay <ploffay@redhat.com> * cleanup Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Bump otel version that has all fixes Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Bump OTEL version and remove disabled flag Signed-off-by: Pavol Loffay <ploffay@redhat.com> * change import Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Always override arrays Signed-off-by: Pavol Loffay <ploffay@redhat.com> * Remove comment Signed-off-by: Pavol Loffay <ploffay@redhat.com>
1 parent 8611137 commit e43a11d

16 files changed

+472
-89
lines changed

cmd/opentelemetry-collector/app/defaults/default_config.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func Config(storageType string, zipkinHostPort string, factories config.Factorie
5353
Extensions: configmodels.Extensions{"health_check": hc},
5454
Service: configmodels.Service{
5555
Extensions: []string{"health_check"},
56-
Pipelines: map[string]*configmodels.Pipeline{
56+
Pipelines: configmodels.Pipelines{
5757
"traces": {
5858
InputType: configmodels.TracesDataType,
5959
Receivers: recTypes,

cmd/opentelemetry-collector/app/defaults/default_config_test.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,14 @@ func TestDefaultConfig(t *testing.T) {
3838
storageType string
3939
zipkinHostPort string
4040
exporterTypes []string
41-
pipeline map[string]*configmodels.Pipeline
41+
pipeline configmodels.Pipelines
4242
err string
4343
}{
4444
{
4545
storageType: "elasticsearch",
4646
zipkinHostPort: disabledHostPort,
4747
exporterTypes: []string{elasticsearch.TypeStr},
48-
pipeline: map[string]*configmodels.Pipeline{
48+
pipeline: configmodels.Pipelines{
4949
"traces": {
5050
InputType: configmodels.TracesDataType,
5151
Receivers: []string{"jaeger"},
@@ -57,7 +57,7 @@ func TestDefaultConfig(t *testing.T) {
5757
storageType: "cassandra",
5858
zipkinHostPort: disabledHostPort,
5959
exporterTypes: []string{cassandra.TypeStr},
60-
pipeline: map[string]*configmodels.Pipeline{
60+
pipeline: configmodels.Pipelines{
6161
"traces": {
6262
InputType: configmodels.TracesDataType,
6363
Receivers: []string{"jaeger"},
@@ -69,7 +69,7 @@ func TestDefaultConfig(t *testing.T) {
6969
storageType: "kafka",
7070
zipkinHostPort: disabledHostPort,
7171
exporterTypes: []string{kafka.TypeStr},
72-
pipeline: map[string]*configmodels.Pipeline{
72+
pipeline: configmodels.Pipelines{
7373
"traces": {
7474
InputType: configmodels.TracesDataType,
7575
Receivers: []string{"jaeger"},
@@ -81,7 +81,7 @@ func TestDefaultConfig(t *testing.T) {
8181
storageType: "cassandra,elasticsearch",
8282
zipkinHostPort: disabledHostPort,
8383
exporterTypes: []string{cassandra.TypeStr, elasticsearch.TypeStr},
84-
pipeline: map[string]*configmodels.Pipeline{
84+
pipeline: configmodels.Pipelines{
8585
"traces": {
8686
InputType: configmodels.TracesDataType,
8787
Receivers: []string{"jaeger"},
@@ -93,7 +93,7 @@ func TestDefaultConfig(t *testing.T) {
9393
storageType: "cassandra",
9494
zipkinHostPort: ":9411",
9595
exporterTypes: []string{cassandra.TypeStr},
96-
pipeline: map[string]*configmodels.Pipeline{
96+
pipeline: configmodels.Pipelines{
9797
"traces": {
9898
InputType: configmodels.TracesDataType,
9999
Receivers: []string{"jaeger", "zipkin"},
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright (c) 2020 The Jaeger Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package defaults
16+
17+
import (
18+
"github.com/imdario/mergo"
19+
"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
20+
)
21+
22+
// MergeConfigs merges two configs.
23+
// The src is merged into dst.
24+
func MergeConfigs(dst, src *configmodels.Config) error {
25+
if src == nil {
26+
return nil
27+
}
28+
err := mergo.Merge(dst, src,
29+
mergo.WithOverride)
30+
if err != nil {
31+
return err
32+
}
33+
return nil
34+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
// Copyright (c) 2020 The Jaeger Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package defaults
16+
17+
import (
18+
"fmt"
19+
"testing"
20+
21+
"github.com/open-telemetry/opentelemetry-collector/config"
22+
"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
23+
"github.com/open-telemetry/opentelemetry-collector/processor/attributesprocessor"
24+
"github.com/open-telemetry/opentelemetry-collector/processor/batchprocessor"
25+
"github.com/open-telemetry/opentelemetry-collector/receiver"
26+
"github.com/open-telemetry/opentelemetry-collector/receiver/jaegerreceiver"
27+
"github.com/open-telemetry/opentelemetry-collector/receiver/zipkinreceiver"
28+
"github.com/spf13/viper"
29+
"github.com/stretchr/testify/assert"
30+
"github.com/stretchr/testify/require"
31+
32+
"github.com/jaegertracing/jaeger/cmd/opentelemetry-collector/app/exporter/elasticsearch"
33+
jConfig "github.com/jaegertracing/jaeger/pkg/config"
34+
)
35+
36+
func TestMergeConfigs_nil(t *testing.T) {
37+
cfg := &configmodels.Config{
38+
Receivers: configmodels.Receivers{
39+
"jaeger": &jaegerreceiver.Config{
40+
RemoteSampling: &jaegerreceiver.RemoteSamplingConfig{StrategyFile: "file.json"},
41+
},
42+
},
43+
}
44+
err := MergeConfigs(cfg, nil)
45+
require.NoError(t, err)
46+
assert.Equal(t, cfg, cfg)
47+
}
48+
49+
func TestMergeConfigs(t *testing.T) {
50+
cfg := &configmodels.Config{
51+
Receivers: configmodels.Receivers{
52+
"jaeger": &jaegerreceiver.Config{
53+
Protocols: map[string]*receiver.SecureReceiverSettings{
54+
"grpc": {ReceiverSettings: configmodels.ReceiverSettings{Endpoint: "def"}},
55+
"thrift_compact": {ReceiverSettings: configmodels.ReceiverSettings{Endpoint: "def"}},
56+
},
57+
},
58+
},
59+
Processors: configmodels.Processors{
60+
"batch": &batchprocessor.Config{
61+
SendBatchSize: uint32(160),
62+
},
63+
},
64+
Service: configmodels.Service{
65+
Extensions: []string{"def", "def2"},
66+
Pipelines: configmodels.Pipelines{
67+
"traces": &configmodels.Pipeline{
68+
Receivers: []string{"jaeger"},
69+
Processors: []string{"batch"},
70+
},
71+
},
72+
},
73+
}
74+
overrideCfg := &configmodels.Config{
75+
Receivers: configmodels.Receivers{
76+
"jaeger": &jaegerreceiver.Config{
77+
Protocols: map[string]*receiver.SecureReceiverSettings{
78+
"grpc": {ReceiverSettings: configmodels.ReceiverSettings{Endpoint: "master_jaeger_url"}},
79+
},
80+
},
81+
"zipkin": &zipkinreceiver.Config{
82+
ReceiverSettings: configmodels.ReceiverSettings{
83+
Endpoint: "master_zipkin_url",
84+
},
85+
},
86+
},
87+
Processors: configmodels.Processors{
88+
"attributes": &attributesprocessor.Config{
89+
Actions: []attributesprocessor.ActionKeyValue{{Key: "foo"}},
90+
},
91+
},
92+
Service: configmodels.Service{
93+
Extensions: []string{"def", "master1", "master2"},
94+
Pipelines: configmodels.Pipelines{
95+
"traces": &configmodels.Pipeline{
96+
Receivers: []string{"jaeger", "zipkin"},
97+
Processors: []string{"attributes"},
98+
},
99+
"traces/2": &configmodels.Pipeline{
100+
Processors: []string{"example"},
101+
},
102+
},
103+
},
104+
}
105+
expected := &configmodels.Config{
106+
Receivers: configmodels.Receivers{
107+
"jaeger": &jaegerreceiver.Config{
108+
Protocols: map[string]*receiver.SecureReceiverSettings{
109+
"grpc": {ReceiverSettings: configmodels.ReceiverSettings{Endpoint: "master_jaeger_url"}},
110+
"thrift_compact": {ReceiverSettings: configmodels.ReceiverSettings{Endpoint: "def"}},
111+
},
112+
},
113+
"zipkin": &zipkinreceiver.Config{
114+
ReceiverSettings: configmodels.ReceiverSettings{
115+
Endpoint: "master_zipkin_url",
116+
},
117+
},
118+
},
119+
Processors: configmodels.Processors{
120+
"batch": &batchprocessor.Config{
121+
SendBatchSize: uint32(160),
122+
},
123+
"attributes": &attributesprocessor.Config{
124+
Actions: []attributesprocessor.ActionKeyValue{{Key: "foo"}},
125+
},
126+
},
127+
Service: configmodels.Service{
128+
Extensions: []string{"def", "master1", "master2"},
129+
Pipelines: configmodels.Pipelines{
130+
"traces": &configmodels.Pipeline{
131+
Receivers: []string{"jaeger", "zipkin"},
132+
Processors: []string{"attributes"},
133+
},
134+
"traces/2": &configmodels.Pipeline{
135+
Processors: []string{"example"},
136+
},
137+
},
138+
},
139+
}
140+
err := MergeConfigs(cfg, overrideCfg)
141+
require.NoError(t, err)
142+
assert.Equal(t, expected, cfg)
143+
}
144+
145+
func TestMergeConfigFiles(t *testing.T) {
146+
testFiles := []string{"emptyoverride", "addprocessor", "multiplecomponents"}
147+
v, _ := jConfig.Viperize(elasticsearch.DefaultOptions().AddFlags)
148+
cmpts := Components(v)
149+
for _, f := range testFiles {
150+
t.Run(f, func(t *testing.T) {
151+
cfg, err := loadConfig(cmpts, fmt.Sprintf("testdata/%s.yaml", f))
152+
require.NoError(t, err)
153+
override, err := loadConfig(cmpts, fmt.Sprintf("testdata/%s-override.yaml", f))
154+
require.NoError(t, err)
155+
merged, err := loadConfig(cmpts, fmt.Sprintf("testdata/%s-merged.yaml", f))
156+
require.NoError(t, err)
157+
err = MergeConfigs(cfg, override)
158+
require.NoError(t, err)
159+
assert.Equal(t, merged, cfg)
160+
})
161+
}
162+
}
163+
164+
func loadConfig(factories config.Factories, file string) (*configmodels.Config, error) {
165+
// config.Load fails to load an empty config
166+
if file == "testdata/emptyoverride-override.yaml" {
167+
return &configmodels.Config{}, nil
168+
}
169+
v := viper.New()
170+
v.SetConfigFile(file)
171+
err := v.ReadInConfig()
172+
if err != nil {
173+
return nil, fmt.Errorf("error loading config file %q: %v", file, err)
174+
}
175+
return config.Load(v, factories)
176+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
receivers:
2+
jaeger:
3+
protocols:
4+
grpc:
5+
thrift_compact:
6+
thrift_binary:
7+
8+
processors:
9+
queued_retry: {}
10+
batch:
11+
timeout: 5s
12+
13+
exporters:
14+
jaeger_elasticsearch:
15+
16+
service:
17+
pipelines:
18+
traces:
19+
receivers: [jaeger]
20+
processors: [batch]
21+
exporters: [jaeger_elasticsearch]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# TODO OTEL config.Load() fails if there are no receiver/exporters
2+
# https://github.com/open-telemetry/opentelemetry-collector/issues/888
3+
receivers:
4+
jaeger:
5+
protocols:
6+
grpc:
7+
exporters:
8+
jaeger_elasticsearch:
9+
# TODO These two properties have to be overridden here if they were overridden in addprocessor.yaml
10+
# If we do not override them here the default values from viper will override the values from "default/src" config.
11+
# However in reality the default config is created from viper, hence when OTEL configuration is created it
12+
# uses viper to create default values and then it sets the values from config.
13+
# The viper is not used in tests hence we have to override it here.
14+
#num_shards: 1
15+
#num_replicas: 0
16+
17+
processors:
18+
batch:
19+
timeout: 5s
20+
21+
service:
22+
pipelines:
23+
traces:
24+
processors: [batch]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
receivers:
2+
jaeger:
3+
protocols:
4+
grpc:
5+
thrift_compact:
6+
thrift_binary:
7+
8+
processors:
9+
queued_retry: {}
10+
11+
exporters:
12+
jaeger_elasticsearch:
13+
14+
service:
15+
pipelines:
16+
traces:
17+
receivers: [jaeger]
18+
processors: [queued_retry]
19+
exporters: [jaeger_elasticsearch]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
receivers:
2+
jaeger:
3+
protocols:
4+
grpc:
5+
thrift_compact:
6+
thrift_binary:
7+
8+
processors:
9+
queued_retry: {}
10+
11+
exporters:
12+
jaeger_elasticsearch:
13+
server_urls: http://localhost:9200
14+
num_shards: 1
15+
num_replicas: 0
16+
17+
service:
18+
pipelines:
19+
traces:
20+
receivers: [jaeger]
21+
processors: [queued_retry]
22+
exporters: [jaeger_elasticsearch]
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
receivers:
2+
jaeger:
3+
protocols:
4+
grpc:
5+
thrift_compact:
6+
thrift_binary:
7+
8+
processors:
9+
queued_retry: {}
10+
11+
exporters:
12+
jaeger_elasticsearch:
13+
server_urls: http://localhost:9200
14+
num_shards: 1
15+
num_replicas: 0
16+
17+
service:
18+
pipelines:
19+
traces:
20+
receivers: [jaeger]
21+
processors: [queued_retry]
22+
exporters: [jaeger_elasticsearch]

0 commit comments

Comments
 (0)