Skip to content
This repository was archived by the owner on Jan 11, 2023. It is now read-only.

Commit d7fbba8

Browse files
karataliukarataliu
committed
Add cloud-controller-manager support
1 parent 56f5aaf commit d7fbba8

14 files changed

+152
-8
lines changed

‎parts/kubernetesagentcustomdata.yml

+5
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,11 @@ write_files:
128128
KUBELET_FIX_43704_3=""
129129
{{end}}
130130
{{end}}
131+
{{if UseCloudControllerManager }}
132+
CLOUD_PROVIDER=external
133+
{{else}}
134+
CLOUD_PROVIDER=azure
135+
{{end}}
131136

132137
- path: "/etc/systemd/system/kubelet.service"
133138
permissions: "0644"

‎parts/kuberneteskubelet.service

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ ExecStart=/usr/bin/docker run \
4444
--cluster-dns=${KUBELET_CLUSTER_DNS} \
4545
--cluster-domain=cluster.local \
4646
--node-labels="${KUBELET_NODE_LABELS}" \
47-
--cloud-provider=azure \
47+
--cloud-provider=${CLOUD_PROVIDER} \
4848
--cloud-config=/etc/kubernetes/azure.json \
4949
--azure-container-registry-config=/etc/kubernetes/azure.json \
5050
--network-plugin=${KUBELET_NETWORK_PLUGIN} \

‎parts/kubernetesmaster-cloud-controller-manager.yaml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
apiVersion: "v1"
2+
kind: "Pod"
3+
metadata:
4+
name: "cloud-controller-manager"
5+
namespace: "kube-system"
6+
labels:
7+
tier: control-plane
8+
component: cloud-controller-manager
9+
spec:
10+
hostNetwork: true
11+
containers:
12+
- name: "cloud-controller-manager"
13+
image: "<kubernetesCcmImageSpec>"
14+
command:
15+
- "cloud-controller-manager"
16+
- "--kubeconfig=/var/lib/kubelet/kubeconfig"
17+
- "--allocate-node-cidrs=<allocateNodeCidrs>"
18+
- "--cluster-cidr=<kubeClusterCidr>"
19+
- "--cluster-name=<masterFqdnPrefix>"
20+
- "--cloud-provider=azure"
21+
- "--cloud-config=/etc/kubernetes/azure.json"
22+
- "--leader-elect=true"
23+
# TODO: RBAC support
24+
# - "<kubernetesEnableRbac>"
25+
- "--route-reconciliation-period=<kubernetesCtrlMgrRouteReconciliationPeriod>"
26+
- "--v=2"
27+
volumeMounts:
28+
- name: "etc-kubernetes"
29+
mountPath: "/etc/kubernetes"
30+
- name: "etc-ssl"
31+
mountPath: "/etc/ssl"
32+
readOnly: true
33+
- name: "var-lib-kubelet"
34+
mountPath: "/var/lib/kubelet"
35+
- name: msi
36+
mountPath: "/var/lib/waagent/ManagedIdentity-Settings"
37+
readOnly: true
38+
volumes:
39+
- name: "etc-kubernetes"
40+
hostPath:
41+
path: "/etc/kubernetes"
42+
- name: "etc-ssl"
43+
hostPath:
44+
path: "/etc/ssl"
45+
- name: "var-lib-kubelet"
46+
hostPath:
47+
path: "/var/lib/kubelet"
48+
- name: msi
49+
hostPath:
50+
path: "/var/lib/waagent/ManagedIdentity-Settings"

‎parts/kubernetesmastercustomdata.yml

+24
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,15 @@ write_files:
100100
content: !!binary |
101101
MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR
102102

103+
{{if UseCloudControllerManager }}
104+
- path: /etc/kubernetes/manifests/cloud-controller-manager.yaml
105+
permissions: "0644"
106+
encoding: gzip
107+
owner: "root"
108+
content: !!binary |
109+
MASTER_KUBERNETES_CLOUD_CONTROLLER_MANAGER_B64_GZIP_STR
110+
{{end}}
111+
103112
- path: /etc/kubernetes/manifests/kube-scheduler.yaml
104113
permissions: "0644"
105114
encoding: gzip
@@ -242,6 +251,11 @@ write_files:
242251
{{else}}
243252
KUBELET_REGISTER_SCHEDULABLE={{WrapAsVariable "registerSchedulable"}}
244253
{{end}}
254+
{{if UseCloudControllerManager }}
255+
CLOUD_PROVIDER=external
256+
{{else}}
257+
CLOUD_PROVIDER=azure
258+
{{end}}
245259

246260
- path: "/etc/systemd/system/kubelet.service"
247261
permissions: "0644"
@@ -343,6 +357,16 @@ write_files:
343357
{{end}}
344358
sed -i "s|<etcdApiVersion>|{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
345359

360+
{{if UseCloudControllerManager }}
361+
sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s|<masterFqdnPrefix>|{{WrapAsVariable "masterFqdnPrefix"}}|g; s|<allocateNodeCidrs>|{{WrapAsVariable "allocateNodeCidrs"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g; s|<kubernetesCtrlMgrRouteReconciliationPeriod>|{{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}}|g" \
362+
/etc/kubernetes/manifests/cloud-controller-manager.yaml
363+
364+
sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \
365+
/etc/kubernetes/manifests/kube-controller-manager.yaml
366+
sed -i "/--\(cloud-config\|cloud-provider\)=/d" \
367+
/etc/kubernetes/manifests/kube-apiserver.yaml
368+
{{end}}
369+
346370
- path: "/opt/azure/containers/provision.sh"
347371
permissions: "0744"
348372
encoding: gzip

‎parts/kubernetesmastervars.t

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"kubeConfigCertificate": "[parameters('kubeConfigCertificate')]",
1414
"kubeConfigPrivateKey": "[parameters('kubeConfigPrivateKey')]",
1515
"kubernetesHyperkubeSpec": "[parameters('kubernetesHyperkubeSpec')]",
16+
"kubernetesCcmImageSpec": "[parameters('kubernetesCcmImageSpec')]",
1617
"kubernetesAddonManagerSpec": "[parameters('kubernetesAddonManagerSpec')]",
1718
"kubernetesAddonResizerSpec": "[parameters('kubernetesAddonResizerSpec')]",
1819
"kubernetesDashboardSpec": "[parameters('kubernetesDashboardSpec')]",

‎parts/kubernetesparams.t

+7
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,13 @@
127127
},
128128
"type": "string"
129129
},
130+
"kubernetesCcmImageSpec": {
131+
"defaultValue": "",
132+
"metadata": {
133+
"description": "The container spec for cloud-controller-manager."
134+
},
135+
"type": "string"
136+
},
130137
"kubernetesAddonManagerSpec": {
131138
{{PopulateClassicModeDefaultValue "kubernetesAddonManagerSpec"}}
132139
"metadata": {

‎pkg/acsengine/engine.go

+18-4
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,11 @@ const (
9696
)
9797

9898
var kubernetesManifestYamls = map[string]string{
99-
"MASTER_KUBERNETES_SCHEDULER_B64_GZIP_STR": "kubernetesmaster-kube-scheduler.yaml",
100-
"MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-controller-manager.yaml",
101-
"MASTER_KUBERNETES_APISERVER_B64_GZIP_STR": "kubernetesmaster-kube-apiserver.yaml",
102-
"MASTER_KUBERNETES_ADDON_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-addon-manager.yaml",
99+
"MASTER_KUBERNETES_SCHEDULER_B64_GZIP_STR": "kubernetesmaster-kube-scheduler.yaml",
100+
"MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-controller-manager.yaml",
101+
"MASTER_KUBERNETES_CLOUD_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-cloud-controller-manager.yaml",
102+
"MASTER_KUBERNETES_APISERVER_B64_GZIP_STR": "kubernetesmaster-kube-apiserver.yaml",
103+
"MASTER_KUBERNETES_ADDON_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-addon-manager.yaml",
103104
}
104105

105106
var kubernetesAritfacts = map[string]string{
@@ -543,6 +544,16 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s
543544
if properties.HostedMasterProfile != nil && properties.HostedMasterProfile.FQDN != "" {
544545
addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN)
545546
}
547+
548+
if properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager {
549+
kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"]
550+
if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" {
551+
kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage
552+
}
553+
554+
addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec)
555+
}
556+
546557
addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo)
547558
addValue(parametersMap, "kubeDNSServiceIP", properties.OrchestratorProfile.KubernetesConfig.DNSServiceIP)
548559
addValue(parametersMap, "kubeServiceCidr", properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR)
@@ -1387,6 +1398,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
13871398
}
13881399
return fmt.Sprintf("\"defaultValue\": \"%s\",", val)
13891400
},
1401+
"UseCloudControllerManager": func() bool {
1402+
return cs.Properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager
1403+
},
13901404
// inspired by http://stackoverflow.com/questions/18276173/calling-a-template-with-several-pipeline-parameters/18276968#18276968
13911405
"dict": func(values ...interface{}) (map[string]interface{}, error) {
13921406
if len(values)%2 != 0 {

‎pkg/acsengine/k8s_versions.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ import (
99
// KubeConfigs represents Docker images used for Kubernetes components based on Kubernetes versions (major.minor.patch)
1010
var KubeConfigs = map[string]map[string]string{
1111
common.KubernetesVersion1Dot8Dot2: {
12-
"hyperkube": "hyperkube-amd64:v1.8.2",
12+
"hyperkube": "hyperkube-amd64:v1.8.2",
13+
"ccm": "cloud-controller-manager-amd64:v1.8.2",
1314
DefaultDashboardAddonName: "kubernetes-dashboard-amd64:v1.7.1",
1415
"exechealthz": "exechealthz-amd64:1.2",
1516
"addonresizer": "addon-resizer:1.7",
@@ -35,7 +36,8 @@ var KubeConfigs = map[string]map[string]string{
3536
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
3637
},
3738
common.KubernetesVersion1Dot8Dot1: {
38-
"hyperkube": "hyperkube-amd64:v1.8.1",
39+
"hyperkube": "hyperkube-amd64:v1.8.1",
40+
"ccm": "cloud-controller-manager-amd64:v1.8.1",
3941
DefaultDashboardAddonName: "kubernetes-dashboard-amd64:v1.7.1",
4042
"exechealthz": "exechealthz-amd64:1.2",
4143
"addonresizer": "addon-resizer:1.7",
@@ -61,7 +63,8 @@ var KubeConfigs = map[string]map[string]string{
6163
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
6264
},
6365
common.KubernetesVersion1Dot8Dot0: {
64-
"hyperkube": "hyperkube-amd64:v1.8.0",
66+
"hyperkube": "hyperkube-amd64:v1.8.0",
67+
"ccm": "cloud-controller-manager-amd64:v1.8.0",
6568
DefaultDashboardAddonName: "kubernetes-dashboard-amd64:v1.7.1",
6669
"exechealthz": "exechealthz-amd64:1.2",
6770
"addonresizer": "addon-resizer:1.7",

‎pkg/api/converterfromapi.go

+2
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,8 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
664664
vlabs.CloudProviderRateLimitQPS = api.CloudProviderRateLimitQPS
665665
vlabs.UseManagedIdentity = api.UseManagedIdentity
666666
vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage
667+
vlabs.CustomCcmImage = api.CustomCcmImage
668+
vlabs.UseCloudControllerManager = api.UseCloudControllerManager
667669
vlabs.UseInstanceMetadata = api.UseInstanceMetadata
668670
vlabs.EnableRbac = api.EnableRbac
669671
vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs

‎pkg/api/convertertoapi.go

+2
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
606606
api.CloudProviderRateLimitQPS = vlabs.CloudProviderRateLimitQPS
607607
api.UseManagedIdentity = vlabs.UseManagedIdentity
608608
api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage
609+
api.CustomCcmImage = vlabs.CustomCcmImage
610+
api.UseCloudControllerManager = vlabs.UseCloudControllerManager
609611
api.UseInstanceMetadata = vlabs.UseInstanceMetadata
610612
api.EnableRbac = vlabs.EnableRbac
611613
api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs

‎pkg/api/types.go

+2
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ type KubernetesConfig struct {
206206
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"`
207207
UseManagedIdentity bool `json:"useManagedIdentity,omitempty"`
208208
CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"`
209+
CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager
210+
UseCloudControllerManager bool `json:"useCloudControllerManager,omitempty"`
209211
UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"`
210212
EnableRbac bool `json:"enableRbac,omitempty"`
211213
EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"`

‎pkg/api/vlabs/types.go

+2
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,8 @@ type KubernetesConfig struct {
224224
CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"`
225225
UseManagedIdentity bool `json:"useManagedIdentity,omitempty"`
226226
CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"`
227+
CustomCcmImage string `json:"customCcmImage,omitempty"`
228+
UseCloudControllerManager bool `json:"useCloudControllerManager,omitempty"`
227229
UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"`
228230
EnableRbac bool `json:"enableRbac,omitempty"`
229231
EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"`

‎pkg/api/vlabs/validate.go

+12
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,18 @@ func (a *KubernetesConfig) Validate(k8sVersion string) error {
594594
return e
595595
}
596596

597+
var ccmEnabledVersions = map[string]bool{
598+
common.KubernetesVersion1Dot8Dot0: true,
599+
common.KubernetesVersion1Dot8Dot1: true,
600+
common.KubernetesVersion1Dot8Dot2: true,
601+
}
602+
603+
if a.UseCloudControllerManager || a.CustomCcmImage != "" {
604+
if !ccmEnabledVersions[k8sVersion] {
605+
return fmt.Errorf("OrchestratorProfile.KubernetesConfig.UseCloudControllerManager and OrchestratorProfile.KubernetesConfig.CustomCcmImage not available in kubernetes version %s", k8sVersion)
606+
}
607+
}
608+
597609
return nil
598610
}
599611

‎pkg/api/vlabs/validate_test.go

+20
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,26 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
246246
t.Error("should not error when basic backoff and rate limiting are set to true with no options")
247247
}
248248
}
249+
250+
// Tests that apply to pre-1.8 releases
251+
for _, k8sVersion := range []string{common.KubernetesVersion1Dot5Dot8, common.KubernetesVersion1Dot6Dot11, common.KubernetesVersion1Dot7Dot7} {
252+
c := KubernetesConfig{
253+
UseCloudControllerManager: true,
254+
}
255+
if err := c.Validate(k8sVersion); err == nil {
256+
t.Error("should error because UseCloudControllerManager is not available before v1.8")
257+
}
258+
}
259+
260+
// Tests that apply to 1.8 and later releases
261+
for _, k8sVersion := range []string{common.KubernetesVersion1Dot8Dot1} {
262+
c := KubernetesConfig{
263+
UseCloudControllerManager: true,
264+
}
265+
if err := c.Validate(k8sVersion); err != nil {
266+
t.Error("should not error because UseCloudControllerManager is available since v1.8")
267+
}
268+
}
249269
}
250270

251271
func Test_Properties_ValidateNetworkPolicy(t *testing.T) {

0 commit comments

Comments
 (0)