diff --git a/parts/kubernetesagentcustomdata.yml b/parts/kubernetesagentcustomdata.yml index 1ee0bb8186..f04fa32537 100644 --- a/parts/kubernetesagentcustomdata.yml +++ b/parts/kubernetesagentcustomdata.yml @@ -128,6 +128,11 @@ write_files: KUBELET_FIX_43704_3="" {{end}} {{end}} +{{if UseCloudControllerManager }} + CLOUD_PROVIDER=external +{{else}} + CLOUD_PROVIDER=azure +{{end}} - path: "/etc/systemd/system/kubelet.service" permissions: "0644" diff --git a/parts/kuberneteskubelet.service b/parts/kuberneteskubelet.service index d55dbda43f..ac7bcbf469 100644 --- a/parts/kuberneteskubelet.service +++ b/parts/kuberneteskubelet.service @@ -44,7 +44,7 @@ ExecStart=/usr/bin/docker run \ --cluster-dns=${KUBELET_CLUSTER_DNS} \ --cluster-domain=cluster.local \ --node-labels="${KUBELET_NODE_LABELS}" \ - --cloud-provider=azure \ + --cloud-provider=${CLOUD_PROVIDER} \ --cloud-config=/etc/kubernetes/azure.json \ --azure-container-registry-config=/etc/kubernetes/azure.json \ --network-plugin=${KUBELET_NETWORK_PLUGIN} \ diff --git a/parts/kubernetesmaster-cloud-controller-manager.yaml b/parts/kubernetesmaster-cloud-controller-manager.yaml new file mode 100644 index 0000000000..28a0a0833c --- /dev/null +++ b/parts/kubernetesmaster-cloud-controller-manager.yaml @@ -0,0 +1,50 @@ +apiVersion: "v1" +kind: "Pod" +metadata: + name: "cloud-controller-manager" + namespace: "kube-system" + labels: + tier: control-plane + component: cloud-controller-manager +spec: + hostNetwork: true + containers: + - name: "cloud-controller-manager" + image: "" + command: + - "cloud-controller-manager" + - "--kubeconfig=/var/lib/kubelet/kubeconfig" + - "--allocate-node-cidrs=" + - "--cluster-cidr=" + - "--cluster-name=" + - "--cloud-provider=azure" + - "--cloud-config=/etc/kubernetes/azure.json" + - "--leader-elect=true" +# TODO: RBAC support +# - "" + - "--route-reconciliation-period=" + - "--v=2" + volumeMounts: + - name: "etc-kubernetes" + mountPath: "/etc/kubernetes" + - name: "etc-ssl" + mountPath: "/etc/ssl" + readOnly: true + - name: "var-lib-kubelet" + mountPath: "/var/lib/kubelet" + - name: msi + mountPath: "/var/lib/waagent/ManagedIdentity-Settings" + readOnly: true + volumes: + - name: "etc-kubernetes" + hostPath: + path: "/etc/kubernetes" + - name: "etc-ssl" + hostPath: + path: "/etc/ssl" + - name: "var-lib-kubelet" + hostPath: + path: "/var/lib/kubelet" + - name: msi + hostPath: + path: "/var/lib/waagent/ManagedIdentity-Settings" diff --git a/parts/kubernetesmastercustomdata.yml b/parts/kubernetesmastercustomdata.yml index 646ef255db..db73c13bbc 100644 --- a/parts/kubernetesmastercustomdata.yml +++ b/parts/kubernetesmastercustomdata.yml @@ -100,6 +100,15 @@ write_files: content: !!binary | MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR +{{if UseCloudControllerManager }} +- path: /etc/kubernetes/manifests/cloud-controller-manager.yaml + permissions: "0644" + encoding: gzip + owner: "root" + content: !!binary | + MASTER_KUBERNETES_CLOUD_CONTROLLER_MANAGER_B64_GZIP_STR +{{end}} + - path: /etc/kubernetes/manifests/kube-scheduler.yaml permissions: "0644" encoding: gzip @@ -242,6 +251,11 @@ write_files: {{else}} KUBELET_REGISTER_SCHEDULABLE={{WrapAsVariable "registerSchedulable"}} {{end}} +{{if UseCloudControllerManager }} + CLOUD_PROVIDER=external +{{else}} + CLOUD_PROVIDER=azure +{{end}} - path: "/etc/systemd/system/kubelet.service" permissions: "0644" @@ -343,6 +357,16 @@ write_files: {{end}} sed -i "s||{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" +{{if UseCloudControllerManager }} + sed -i "s||{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s||{{WrapAsVariable "masterFqdnPrefix"}}|g; s||{{WrapAsVariable "allocateNodeCidrs"}}|g; s||{{WrapAsVariable "kubeClusterCidr"}}|g; s||{{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}}|g" \ + /etc/kubernetes/manifests/cloud-controller-manager.yaml + + sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \ + /etc/kubernetes/manifests/kube-controller-manager.yaml + sed -i "/--\(cloud-config\|cloud-provider\)=/d" \ + /etc/kubernetes/manifests/kube-apiserver.yaml +{{end}} + - path: "/opt/azure/containers/provision.sh" permissions: "0744" encoding: gzip diff --git a/parts/kubernetesmastervars.t b/parts/kubernetesmastervars.t index 62e5262eb4..266384169d 100644 --- a/parts/kubernetesmastervars.t +++ b/parts/kubernetesmastervars.t @@ -13,6 +13,7 @@ "kubeConfigCertificate": "[parameters('kubeConfigCertificate')]", "kubeConfigPrivateKey": "[parameters('kubeConfigPrivateKey')]", "kubernetesHyperkubeSpec": "[parameters('kubernetesHyperkubeSpec')]", + "kubernetesCcmImageSpec": "[parameters('kubernetesCcmImageSpec')]", "kubernetesAddonManagerSpec": "[parameters('kubernetesAddonManagerSpec')]", "kubernetesAddonResizerSpec": "[parameters('kubernetesAddonResizerSpec')]", "kubernetesDashboardSpec": "[parameters('kubernetesDashboardSpec')]", diff --git a/parts/kubernetesparams.t b/parts/kubernetesparams.t index 7f7d28a273..1c2a6b6687 100644 --- a/parts/kubernetesparams.t +++ b/parts/kubernetesparams.t @@ -127,6 +127,13 @@ }, "type": "string" }, + "kubernetesCcmImageSpec": { + "defaultValue": "", + "metadata": { + "description": "The container spec for cloud-controller-manager." + }, + "type": "string" + }, "kubernetesAddonManagerSpec": { {{PopulateClassicModeDefaultValue "kubernetesAddonManagerSpec"}} "metadata": { diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 61ea203531..fc89cf969c 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -96,10 +96,11 @@ const ( ) var kubernetesManifestYamls = map[string]string{ - "MASTER_KUBERNETES_SCHEDULER_B64_GZIP_STR": "kubernetesmaster-kube-scheduler.yaml", - "MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-controller-manager.yaml", - "MASTER_KUBERNETES_APISERVER_B64_GZIP_STR": "kubernetesmaster-kube-apiserver.yaml", - "MASTER_KUBERNETES_ADDON_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-addon-manager.yaml", + "MASTER_KUBERNETES_SCHEDULER_B64_GZIP_STR": "kubernetesmaster-kube-scheduler.yaml", + "MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-controller-manager.yaml", + "MASTER_KUBERNETES_CLOUD_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-cloud-controller-manager.yaml", + "MASTER_KUBERNETES_APISERVER_B64_GZIP_STR": "kubernetesmaster-kube-apiserver.yaml", + "MASTER_KUBERNETES_ADDON_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-addon-manager.yaml", } var kubernetesAritfacts = map[string]string{ @@ -543,6 +544,16 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s if properties.HostedMasterProfile != nil && properties.HostedMasterProfile.FQDN != "" { addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN) } + + if properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager { + kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"] + if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" { + kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage + } + + addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec) + } + addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo) addValue(parametersMap, "kubeDNSServiceIP", properties.OrchestratorProfile.KubernetesConfig.DNSServiceIP) addValue(parametersMap, "kubeServiceCidr", properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR) @@ -1387,6 +1398,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat } return fmt.Sprintf("\"defaultValue\": \"%s\",", val) }, + "UseCloudControllerManager": func() bool { + return cs.Properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager != nil && *cs.Properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager + }, // inspired by http://stackoverflow.com/questions/18276173/calling-a-template-with-several-pipeline-parameters/18276968#18276968 "dict": func(values ...interface{}) (map[string]interface{}, error) { if len(values)%2 != 0 { diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index b53245a229..8b75d2e570 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -9,7 +9,8 @@ import ( // KubeConfigs represents Docker images used for Kubernetes components based on Kubernetes versions (major.minor.patch) var KubeConfigs = map[string]map[string]string{ common.KubernetesVersion1Dot8Dot2: { - "hyperkube": "hyperkube-amd64:v1.8.2", + "hyperkube": "hyperkube-amd64:v1.8.2", + "ccm": "cloud-controller-manager-amd64:v1.8.2", DefaultDashboardAddonName: "kubernetes-dashboard-amd64:v1.7.1", "exechealthz": "exechealthz-amd64:1.2", "addonresizer": "addon-resizer:1.7", @@ -35,7 +36,8 @@ var KubeConfigs = map[string]map[string]string{ "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, common.KubernetesVersion1Dot8Dot1: { - "hyperkube": "hyperkube-amd64:v1.8.1", + "hyperkube": "hyperkube-amd64:v1.8.1", + "ccm": "cloud-controller-manager-amd64:v1.8.1", DefaultDashboardAddonName: "kubernetes-dashboard-amd64:v1.7.1", "exechealthz": "exechealthz-amd64:1.2", "addonresizer": "addon-resizer:1.7", @@ -61,7 +63,8 @@ var KubeConfigs = map[string]map[string]string{ "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, common.KubernetesVersion1Dot8Dot0: { - "hyperkube": "hyperkube-amd64:v1.8.0", + "hyperkube": "hyperkube-amd64:v1.8.0", + "ccm": "cloud-controller-manager-amd64:v1.8.0", DefaultDashboardAddonName: "kubernetes-dashboard-amd64:v1.7.1", "exechealthz": "exechealthz-amd64:1.2", "addonresizer": "addon-resizer:1.7", diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 3f1593803e..c0b42c40c2 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -664,6 +664,8 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet vlabs.CloudProviderRateLimitQPS = api.CloudProviderRateLimitQPS vlabs.UseManagedIdentity = api.UseManagedIdentity vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage + vlabs.CustomCcmImage = api.CustomCcmImage + vlabs.UseCloudControllerManager = api.UseCloudControllerManager vlabs.UseInstanceMetadata = api.UseInstanceMetadata vlabs.EnableRbac = api.EnableRbac vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 442d55f7f5..6e870bebcf 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -606,6 +606,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.CloudProviderRateLimitQPS = vlabs.CloudProviderRateLimitQPS api.UseManagedIdentity = vlabs.UseManagedIdentity api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage + api.CustomCcmImage = vlabs.CustomCcmImage + api.UseCloudControllerManager = vlabs.UseCloudControllerManager api.UseInstanceMetadata = vlabs.UseInstanceMetadata api.EnableRbac = vlabs.EnableRbac api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs diff --git a/pkg/api/types.go b/pkg/api/types.go index 89a6d1e284..4486303fd2 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -206,6 +206,8 @@ type KubernetesConfig struct { CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` UseManagedIdentity bool `json:"useManagedIdentity,omitempty"` CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` + CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager + UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` EnableRbac bool `json:"enableRbac,omitempty"` EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 421428d6c2..318e0ea2dc 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -224,6 +224,8 @@ type KubernetesConfig struct { CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` UseManagedIdentity bool `json:"useManagedIdentity,omitempty"` CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` + CustomCcmImage string `json:"customCcmImage,omitempty"` + UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` EnableRbac bool `json:"enableRbac,omitempty"` EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index cf404e062c..02f074ce21 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -594,6 +594,18 @@ func (a *KubernetesConfig) Validate(k8sVersion string) error { return e } + var ccmEnabledVersions = map[string]bool{ + common.KubernetesVersion1Dot8Dot0: true, + common.KubernetesVersion1Dot8Dot1: true, + common.KubernetesVersion1Dot8Dot2: true, + } + + if a.UseCloudControllerManager != nil && *a.UseCloudControllerManager || a.CustomCcmImage != "" { + if !ccmEnabledVersions[k8sVersion] { + return fmt.Errorf("OrchestratorProfile.KubernetesConfig.UseCloudControllerManager and OrchestratorProfile.KubernetesConfig.CustomCcmImage not available in kubernetes version %s", k8sVersion) + } + } + return nil } diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index e315ce5ccc..c13f4c9d9b 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -246,6 +246,27 @@ func Test_KubernetesConfig_Validate(t *testing.T) { t.Error("should not error when basic backoff and rate limiting are set to true with no options") } } + + trueVal := true + // Tests that apply to pre-1.8 releases + for _, k8sVersion := range []string{common.KubernetesVersion1Dot5Dot8, common.KubernetesVersion1Dot6Dot11, common.KubernetesVersion1Dot7Dot7} { + c := KubernetesConfig{ + UseCloudControllerManager: &trueVal, + } + if err := c.Validate(k8sVersion); err == nil { + t.Error("should error because UseCloudControllerManager is not available before v1.8") + } + } + + // Tests that apply to 1.8 and later releases + for _, k8sVersion := range []string{common.KubernetesVersion1Dot8Dot1} { + c := KubernetesConfig{ + UseCloudControllerManager: &trueVal, + } + if err := c.Validate(k8sVersion); err != nil { + t.Error("should not error because UseCloudControllerManager is available since v1.8") + } + } } func Test_Properties_ValidateNetworkPolicy(t *testing.T) {