Add cloud-controller-manager support for Kubernetes cluster

parts/kubernetesagentcustomdata.yml

@@ -128,6 +128,11 @@ write_files:
     KUBELET_FIX_43704_3=""
   {{end}}
 {{end}}
+{{if UseCloudControllerManager }}
+    CLOUD_PROVIDER=external
+{{else}}
+    CLOUD_PROVIDER=azure
+{{end}}
 
 - path: "/etc/systemd/system/kubelet.service"
   permissions: "0644"

parts/kuberneteskubelet.service

@@ -44,7 +44,7 @@ ExecStart=/usr/bin/docker run \
         --cluster-dns=${KUBELET_CLUSTER_DNS} \
         --cluster-domain=cluster.local \
         --node-labels="${KUBELET_NODE_LABELS}" \
-        --cloud-provider=azure \
+        --cloud-provider=${CLOUD_PROVIDER} \
         --cloud-config=/etc/kubernetes/azure.json \
         --azure-container-registry-config=/etc/kubernetes/azure.json \
         --network-plugin=${KUBELET_NETWORK_PLUGIN} \

parts/kubernetesmaster-cloud-controller-manager.yaml

@@ -0,0 +1,50 @@
+apiVersion: "v1"
+kind: "Pod"
+metadata:
+  name: "cloud-controller-manager"
+  namespace: "kube-system"
+  labels:
+    tier: control-plane
+    component: cloud-controller-manager
+spec:
+  hostNetwork: true
+  containers:
+    - name: "cloud-controller-manager"
+      image: "<kubernetesCcmImageSpec>"
+      command: 
+        - "cloud-controller-manager"
+        - "--kubeconfig=/var/lib/kubelet/kubeconfig"
+        - "--allocate-node-cidrs=<allocateNodeCidrs>"
+        - "--cluster-cidr=<kubeClusterCidr>"
+        - "--cluster-name=<masterFqdnPrefix>"
+        - "--cloud-provider=azure"
+        - "--cloud-config=/etc/kubernetes/azure.json"
+        - "--leader-elect=true"
+# TODO: RBAC support
+#        - "<kubernetesEnableRbac>"
+        - "--route-reconciliation-period=<kubernetesCtrlMgrRouteReconciliationPeriod>"
+        - "--v=2"
+      volumeMounts: 
+        - name: "etc-kubernetes"
+          mountPath: "/etc/kubernetes"
+        - name: "etc-ssl"
+          mountPath: "/etc/ssl"
+          readOnly: true
+        - name: "var-lib-kubelet"
+          mountPath: "/var/lib/kubelet"
+        - name: msi
+          mountPath: "/var/lib/waagent/ManagedIdentity-Settings"
+          readOnly: true
+  volumes:
+    - name: "etc-kubernetes"
+      hostPath:
+        path: "/etc/kubernetes"
+    - name: "etc-ssl"
+      hostPath:
+        path: "/etc/ssl"
+    - name: "var-lib-kubelet"
+      hostPath:
+        path: "/var/lib/kubelet"
+    - name: msi
+      hostPath:
+        path: "/var/lib/waagent/ManagedIdentity-Settings"

parts/kubernetesmastercustomdata.yml

@@ -100,6 +100,15 @@ write_files:
   content: !!binary |
     MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR
 
+{{if UseCloudControllerManager }}
+- path: /etc/kubernetes/manifests/cloud-controller-manager.yaml
+  permissions: "0644"
+  encoding: gzip
+  owner: "root"
+  content: !!binary |
+    MASTER_KUBERNETES_CLOUD_CONTROLLER_MANAGER_B64_GZIP_STR
+{{end}}
+
 - path: /etc/kubernetes/manifests/kube-scheduler.yaml
   permissions: "0644"
   encoding: gzip
@@ -242,6 +251,11 @@ write_files:
 {{else}}
     KUBELET_REGISTER_SCHEDULABLE={{WrapAsVariable "registerSchedulable"}}
 {{end}}
+{{if UseCloudControllerManager }}
+    CLOUD_PROVIDER=external
+{{else}}
+    CLOUD_PROVIDER=azure
+{{end}}
 
 - path: "/etc/systemd/system/kubelet.service"
   permissions: "0644"
@@ -343,6 +357,16 @@ write_files:
 {{end}}
     sed -i "s|<etcdApiVersion>|{{ .OrchestratorProfile.GetAPIServerEtcdAPIVersion }}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml"
 
+{{if UseCloudControllerManager }}
+    sed -i "s|<kubernetesCcmImageSpec>|{{WrapAsVariable "kubernetesCcmImageSpec"}}|g; s|<masterFqdnPrefix>|{{WrapAsVariable "masterFqdnPrefix"}}|g; s|<allocateNodeCidrs>|{{WrapAsVariable "allocateNodeCidrs"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g; s|<kubernetesCtrlMgrRouteReconciliationPeriod>|{{WrapAsVariable "kubernetesCtrlMgrRouteReconciliationPeriod"}}|g" \
+      /etc/kubernetes/manifests/cloud-controller-manager.yaml
+
+    sed -i "/--\(cloud-config\|cloud-provider\|route-reconciliation-period\)=/d" \
+      /etc/kubernetes/manifests/kube-controller-manager.yaml
+    sed -i "/--\(cloud-config\|cloud-provider\)=/d" \
+      /etc/kubernetes/manifests/kube-apiserver.yaml
+{{end}}
+
 - path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
   encoding: gzip

parts/kubernetesmastervars.t

@@ -13,6 +13,7 @@
     "kubeConfigCertificate": "[parameters('kubeConfigCertificate')]",
     "kubeConfigPrivateKey": "[parameters('kubeConfigPrivateKey')]",
     "kubernetesHyperkubeSpec": "[parameters('kubernetesHyperkubeSpec')]",
+    "kubernetesCcmImageSpec": "[parameters('kubernetesCcmImageSpec')]",
     "kubernetesAddonManagerSpec": "[parameters('kubernetesAddonManagerSpec')]",
     "kubernetesAddonResizerSpec": "[parameters('kubernetesAddonResizerSpec')]",
     "kubernetesDashboardSpec": "[parameters('kubernetesDashboardSpec')]",

parts/kubernetesparams.t

@@ -127,6 +127,13 @@
       },
       "type": "string"
     },
+    "kubernetesCcmImageSpec": {
+      "defaultValue": "",
+      "metadata": {
+        "description": "The container spec for cloud-controller-manager."
+      },
+      "type": "string"
+    },
     "kubernetesAddonManagerSpec": {
       {{PopulateClassicModeDefaultValue "kubernetesAddonManagerSpec"}}
       "metadata": {

pkg/acsengine/engine.go

@@ -96,10 +96,11 @@ const (
 )
 
 var kubernetesManifestYamls = map[string]string{
-	"MASTER_KUBERNETES_SCHEDULER_B64_GZIP_STR":          "kubernetesmaster-kube-scheduler.yaml",
-	"MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-kube-controller-manager.yaml",
-	"MASTER_KUBERNETES_APISERVER_B64_GZIP_STR":          "kubernetesmaster-kube-apiserver.yaml",
-	"MASTER_KUBERNETES_ADDON_MANAGER_B64_GZIP_STR":      "kubernetesmaster-kube-addon-manager.yaml",
+	"MASTER_KUBERNETES_SCHEDULER_B64_GZIP_STR":                "kubernetesmaster-kube-scheduler.yaml",
+	"MASTER_KUBERNETES_CONTROLLER_MANAGER_B64_GZIP_STR":       "kubernetesmaster-kube-controller-manager.yaml",
+	"MASTER_KUBERNETES_CLOUD_CONTROLLER_MANAGER_B64_GZIP_STR": "kubernetesmaster-cloud-controller-manager.yaml",
+	"MASTER_KUBERNETES_APISERVER_B64_GZIP_STR":                "kubernetesmaster-kube-apiserver.yaml",
+	"MASTER_KUBERNETES_ADDON_MANAGER_B64_GZIP_STR":            "kubernetesmaster-kube-addon-manager.yaml",
 }
 
 var kubernetesAritfacts = map[string]string{
@@ -543,6 +544,16 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s
 		if properties.HostedMasterProfile != nil && properties.HostedMasterProfile.FQDN != "" {
 			addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN)
 		}
+
+		if properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager {
+			kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"]
+			if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" {
+				kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage
+			}
+
+			addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec)
+		}
+
 		addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo)
 		addValue(parametersMap, "kubeDNSServiceIP", properties.OrchestratorProfile.KubernetesConfig.DNSServiceIP)
 		addValue(parametersMap, "kubeServiceCidr", properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR)
@@ -1387,6 +1398,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 			}
 			return fmt.Sprintf("\"defaultValue\": \"%s\",", val)
 		},
+		"UseCloudControllerManager": func() bool {
+			return cs.Properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager
+		},
 		// inspired by http://stackoverflow.com/questions/18276173/calling-a-template-with-several-pipeline-parameters/18276968#18276968
 		"dict": func(values ...interface{}) (map[string]interface{}, error) {
 			if len(values)%2 != 0 {

pkg/acsengine/k8s_versions.go

@@ -9,7 +9,8 @@ import (
 // KubeConfigs represents Docker images used for Kubernetes components based on Kubernetes versions (major.minor.patch)
 var KubeConfigs = map[string]map[string]string{
 	common.KubernetesVersion1Dot8Dot2: {
-		"hyperkube":                 "hyperkube-amd64:v1.8.2",
+		"hyperkube": "hyperkube-amd64:v1.8.2",
+		"ccm":       "cloud-controller-manager-amd64:v1.8.2",
 		DefaultDashboardAddonName:   "kubernetes-dashboard-amd64:v1.7.1",
 		"exechealthz":               "exechealthz-amd64:1.2",
 		"addonresizer":              "addon-resizer:1.7",
@@ -35,7 +36,8 @@ var KubeConfigs = map[string]map[string]string{
 		"gclowthreshold":            strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
 	common.KubernetesVersion1Dot8Dot1: {
-		"hyperkube":                 "hyperkube-amd64:v1.8.1",
+		"hyperkube": "hyperkube-amd64:v1.8.1",
+		"ccm":       "cloud-controller-manager-amd64:v1.8.1",
 		DefaultDashboardAddonName:   "kubernetes-dashboard-amd64:v1.7.1",
 		"exechealthz":               "exechealthz-amd64:1.2",
 		"addonresizer":              "addon-resizer:1.7",
@@ -61,7 +63,8 @@ var KubeConfigs = map[string]map[string]string{
 		"gclowthreshold":            strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
 	common.KubernetesVersion1Dot8Dot0: {
-		"hyperkube":                 "hyperkube-amd64:v1.8.0",
+		"hyperkube": "hyperkube-amd64:v1.8.0",
+		"ccm":       "cloud-controller-manager-amd64:v1.8.0",
 		DefaultDashboardAddonName:   "kubernetes-dashboard-amd64:v1.7.1",
 		"exechealthz":               "exechealthz-amd64:1.2",
 		"addonresizer":              "addon-resizer:1.7",

pkg/api/converterfromapi.go

@@ -664,6 +664,8 @@ func convertKubernetesConfigToVLabs(api *KubernetesConfig, vlabs *vlabs.Kubernet
 	vlabs.CloudProviderRateLimitQPS = api.CloudProviderRateLimitQPS
 	vlabs.UseManagedIdentity = api.UseManagedIdentity
 	vlabs.CustomHyperkubeImage = api.CustomHyperkubeImage
+	vlabs.CustomCcmImage = api.CustomCcmImage
+	vlabs.UseCloudControllerManager = api.UseCloudControllerManager
 	vlabs.UseInstanceMetadata = api.UseInstanceMetadata
 	vlabs.EnableRbac = api.EnableRbac
 	vlabs.EnableAggregatedAPIs = api.EnableAggregatedAPIs

pkg/api/convertertoapi.go

@@ -606,6 +606,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes
 	api.CloudProviderRateLimitQPS = vlabs.CloudProviderRateLimitQPS
 	api.UseManagedIdentity = vlabs.UseManagedIdentity
 	api.CustomHyperkubeImage = vlabs.CustomHyperkubeImage
+	api.CustomCcmImage = vlabs.CustomCcmImage
+	api.UseCloudControllerManager = vlabs.UseCloudControllerManager
 	api.UseInstanceMetadata = vlabs.UseInstanceMetadata
 	api.EnableRbac = vlabs.EnableRbac
 	api.EnableAggregatedAPIs = vlabs.EnableAggregatedAPIs

pkg/api/types.go

@@ -206,6 +206,8 @@ type KubernetesConfig struct {
 	CloudProviderRateLimitBucket     int               `json:"cloudProviderRateLimitBucket,omitempty"`
 	UseManagedIdentity               bool              `json:"useManagedIdentity,omitempty"`
 	CustomHyperkubeImage             string            `json:"customHyperkubeImage,omitempty"`
+	CustomCcmImage                   string            `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager
+	UseCloudControllerManager        bool              `json:"useCloudControllerManager,omitempty"`
 	UseInstanceMetadata              *bool             `json:"useInstanceMetadata,omitempty"`
 	EnableRbac                       bool              `json:"enableRbac,omitempty"`
 	EnableAggregatedAPIs             bool              `json:"enableAggregatedAPIs,omitempty"`

pkg/api/vlabs/types.go

@@ -224,6 +224,8 @@ type KubernetesConfig struct {
 	CloudProviderRateLimitBucket     int               `json:"cloudProviderRateLimitBucket,omitempty"`
 	UseManagedIdentity               bool              `json:"useManagedIdentity,omitempty"`
 	CustomHyperkubeImage             string            `json:"customHyperkubeImage,omitempty"`
+	CustomCcmImage                   string            `json:"customCcmImage,omitempty"`
+	UseCloudControllerManager        bool              `json:"useCloudControllerManager,omitempty"`
 	UseInstanceMetadata              *bool             `json:"useInstanceMetadata,omitempty"`
 	EnableRbac                       bool              `json:"enableRbac,omitempty"`
 	EnableAggregatedAPIs             bool              `json:"enableAggregatedAPIs,omitempty"`

pkg/api/vlabs/validate.go

@@ -594,6 +594,18 @@ func (a *KubernetesConfig) Validate(k8sVersion string) error {
 		return e
 	}
 
+	var ccmEnabledVersions = map[string]bool{
+		common.KubernetesVersion1Dot8Dot0: true,
+		common.KubernetesVersion1Dot8Dot1: true,
+		common.KubernetesVersion1Dot8Dot2: true,
+	}
+
+	if a.UseCloudControllerManager || a.CustomCcmImage != "" {
+		if !ccmEnabledVersions[k8sVersion] {
+			return fmt.Errorf("OrchestratorProfile.KubernetesConfig.UseCloudControllerManager and OrchestratorProfile.KubernetesConfig.CustomCcmImage not available in kubernetes version %s", k8sVersion)
+		}
+	}
+
 	return nil
 }
 

pkg/api/vlabs/validate_test.go

@@ -246,6 +246,26 @@ func Test_KubernetesConfig_Validate(t *testing.T) {
 			t.Error("should not error when basic backoff and rate limiting are set to true with no options")
 		}
 	}
+
+	// Tests that apply to pre-1.8 releases
+	for _, k8sVersion := range []string{common.KubernetesVersion1Dot5Dot8, common.KubernetesVersion1Dot6Dot11, common.KubernetesVersion1Dot7Dot7} {
+		c := KubernetesConfig{
+			UseCloudControllerManager: true,
+		}
+		if err := c.Validate(k8sVersion); err == nil {
+			t.Error("should error because UseCloudControllerManager is not available before v1.8")
+		}
+	}
+
+	// Tests that apply to 1.8 and later releases
+	for _, k8sVersion := range []string{common.KubernetesVersion1Dot8Dot1} {
+		c := KubernetesConfig{
+			UseCloudControllerManager: true,
+		}
+		if err := c.Validate(k8sVersion); err != nil {
+			t.Error("should not error because UseCloudControllerManager is available since v1.8")
+		}
+	}
 }
 
 func Test_Properties_ValidateNetworkPolicy(t *testing.T) {