Revert kubelet server tls bootstrap (#1248)

when the admin upgrades from 1.17 -> 1.18, the admin would do:
- skuba node upgrade apply (loop all control plane nodes and worker nodes)
- skuba addon upgrade apply

then at the first control plane node upgrade, usually the admin checks all the pods in the stable state, however, since the first control node sets `serverTLSBootstrap: true` which means it sends out kubelet server CSR in the cluster but there is no CSR signer right now (kucero installed later in `skuba addon upgrade apply`).

therefore, the metrics-server would be crash loopback state because the kubelet server does not have it's server certificate right now (it seems like the kubelet does not honor in-disk kubelet.crt which is generated by skuba if `serverTLSBootstrap: true`.

Signed-off-by: JenTing Hsiao <jenting.hsiao@suse.com>

internal/pkg/skuba/deployments/ssh/kubelet.go

@@ -18,11 +18,17 @@
 package ssh
 
 import (
+	"crypto/x509"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
+	"strings"
 
+	"github.com/pkg/errors"
+	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil"
 	"sigs.k8s.io/yaml"
 
 	"github.com/SUSE/skuba/internal/pkg/skuba/deployments"
@@ -33,6 +39,7 @@ import (
 
 func init() {
 	stateMap["kubelet.rootcert.upload"] = kubeletUploadRootCert
+	stateMap["kubelet.servercert.create-and-upload"] = kubeletCreateAndUploadServerCert
 	stateMap["kubelet.configure"] = kubeletConfigure
 	stateMap["kubelet.enable"] = kubeletEnable
 }
@@ -55,6 +62,83 @@ func kubeletUploadRootCert(t *Target, data interface{}) error {
 	return nil
 }
 
+func kubeletCreateAndUploadServerCert(t *Target, data interface{}) error {
+	// Read kubelet root ca certificate and key
+	caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(skuba.PkiDir(), kubernetes.KubeletCACertAndKeyBaseName)
+	if err != nil {
+		return errors.Wrap(err, "failure loading kubelet CA certificate authority")
+	}
+
+	host := t.target.Nodename
+	altNames := certutil.AltNames{}
+	if ip := net.ParseIP(host); ip != nil {
+		altNames.IPs = append(altNames.IPs, ip)
+	} else {
+		altNames.DNSNames = append(altNames.DNSNames, host)
+	}
+
+	// Create AltNames with defaults DNSNames/IPs
+	stdout, _, err := t.silentSsh("hostname", "-I")
+	if err != nil {
+		return err
+	}
+	for _, addr := range strings.Split(stdout, " ") {
+		if ip := net.ParseIP(addr); ip != nil {
+			altNames.IPs = append(altNames.IPs, ip)
+		}
+	}
+
+	alternateIPs := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}
+	alternateDNS := []string{"localhost"}
+	if ip := net.ParseIP(t.target.Target); ip != nil {
+		alternateIPs = append(alternateIPs, ip)
+	} else {
+		alternateDNS = append(alternateDNS, t.target.Target)
+	}
+
+	altNames.IPs = append(altNames.IPs, alternateIPs...)
+	altNames.DNSNames = append(altNames.DNSNames, alternateDNS...)
+
+	cfg := &pkiutil.CertConfig{
+		Config: certutil.Config{
+			CommonName: host,
+			AltNames:   altNames,
+			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		},
+	}
+	cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg)
+	if err != nil {
+		return errors.Wrap(err, "couldn't generate kubelet server certificate")
+	}
+
+	// Save kubelet server certificate and key to local temporarily
+	if err := pkiutil.WriteCertAndKey(skuba.PkiDir(), host, cert, key); err != nil {
+		return errors.Wrapf(err, "failure while saving kubelet server %s certificate and key", host)
+	}
+
+	// Upload server certificate and key
+	certPath, keyPath := pkiutil.PathsForCertAndKey(skuba.PkiDir(), host)
+	if err := t.target.UploadFile(certPath, filepath.Join(kubernetes.KubeletCertAndKeyDir, kubernetes.KubeletServerCertName)); err != nil {
+		return err
+	}
+	if err := t.target.UploadFile(keyPath, filepath.Join(kubernetes.KubeletCertAndKeyDir, kubernetes.KubeletServerKeyName)); err != nil {
+		return err
+	}
+	if _, _, err := t.silentSsh("chmod", "0400", filepath.Join(kubernetes.KubeletCertAndKeyDir, kubernetes.KubeletServerKeyName)); err != nil {
+		return err
+	}
+
+	// Remove local temporarily kubelet server certificate and key
+	if err := os.Remove(certPath); err != nil {
+		return err
+	}
+	if err := os.Remove(keyPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func kubeletConfigure(t *Target, data interface{}) error {
 	isSUSE, err := t.target.IsSUSEOS()
 	if err != nil {

internal/pkg/skuba/kubeadm/configmap.go

@@ -25,14 +25,12 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/version"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
-	kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 
 	skubautil "github.com/SUSE/skuba/internal/pkg/skuba/util"
 )
@@ -132,28 +130,11 @@ func RemoveAPIEndpointFromConfigMap(client clientset.Interface, node *corev1.Nod
 
 // UpdateClusterConfigurationWithClusterVersion allows us to set certain configurations during init, but also during upgrades.
 // The configuration that we put here will be consistently set to newly created configurations, and when we upgrade a cluster.
-func UpdateClusterConfigurationWithClusterVersion(initCfg *kubeadmapi.InitConfiguration, clusterVersion *version.Version) ([]byte, error) {
+func UpdateClusterConfigurationWithClusterVersion(initCfg *kubeadmapi.InitConfiguration, clusterVersion *version.Version) {
 	// apiserver
 	setApiserverAdmissionPlugins(initCfg, clusterVersion)
 	setContainerImagesWithClusterVersion(initCfg, clusterVersion)
 	setApiserverArgs(initCfg)
-
-	initCfgContents, err := kubeadmconfigutil.MarshalInitConfigurationToBytes(initCfg, schema.GroupVersion{
-		Group:   "kubeadm.k8s.io",
-		Version: GetKubeadmApisVersion(clusterVersion),
-	})
-	if err != nil {
-		return []byte{}, err
-	}
-
-	// kubelet
-	kubeletManifest := []byte(`---
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-serverTLSBootstrap: true
-`)
-
-	return append(initCfgContents, kubeletManifest...), nil
 }
 
 func setApiserverAdmissionPlugins(initCfg *kubeadmapi.InitConfiguration, clusterVersion *version.Version) {

internal/pkg/skuba/kubeadm/configmap_test.go

@@ -367,10 +367,7 @@ func TestUpdateClusterConfigurationWithClusterVersion(t *testing.T) {
 				}
 				initCfg.APIServer.ControlPlaneComponent.ExtraArgs["enable-admission-plugins"] = currentAdmissionPlugins
 			}
-			_, err := UpdateClusterConfigurationWithClusterVersion(&initCfg, tt.clusterVersion)
-			if err != nil {
-				t.Errorf("expected no error but an error returned: %v", err)
-			}
+			UpdateClusterConfigurationWithClusterVersion(&initCfg, tt.clusterVersion)
 			// Check admission plugins
 			gotAdmissionPlugins := initCfg.APIServer.ControlPlaneComponent.ExtraArgs["enable-admission-plugins"]
 			if gotAdmissionPlugins != expectedAdmissionPlugins {

internal/pkg/skuba/kubernetes/kubelet.go

@@ -45,6 +45,13 @@ const (
 	KubeletCACertName = "kubelet-ca.crt"
 	// KubeletCAKeyName defines kubelet's CA key name
 	KubeletCAKeyName = "kubelet-ca.key"
+
+	// KubeletServerCertAndKeyBaseName defines kubelet server certificate and key base name
+	KubeletServerCertAndKeyBaseName = "kubelet"
+	// KubeletServerCertName defines kubelet server certificate name
+	KubeletServerCertName = "kubelet.crt"
+	// KubeletServerKeyName defines kubelet server key name
+	KubeletServerKeyName = "kubelet.key"
 )
 
 // GenerateKubeletRootCert generates kubelet root CA certificate and key

internal/pkg/skuba/kubernetes/versions.go

@@ -132,7 +132,6 @@ var (
 				Dex:           &AddonVersion{"2.16.0", 6},
 				Gangway:       &AddonVersion{"3.1.0-rev4", 5},
 				MetricsServer: &AddonVersion{"0.3.6", 1},
-				Kucero:        &AddonVersion{"1.1.1", 0},
 				PSP:           &AddonVersion{"", 4},
 			},
 		},

pkg/skuba/actions/cluster/init/init.go

@@ -33,6 +33,7 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 
 	"github.com/SUSE/skuba/internal/pkg/skuba/addons"
 	"github.com/SUSE/skuba/internal/pkg/skuba/kubeadm"
@@ -283,12 +284,14 @@ func writeKubeadmInitConf(initConfiguration InitConfiguration) error {
 	if len(initConfiguration.CloudProvider) > 0 {
 		updateInitConfigurationWithCloudIntegration(&initCfg, initConfiguration)
 	}
-
-	initCfgContents, err := kubeadm.UpdateClusterConfigurationWithClusterVersion(&initCfg, initConfiguration.KubernetesVersion)
+	kubeadm.UpdateClusterConfigurationWithClusterVersion(&initCfg, initConfiguration.KubernetesVersion)
+	initCfgContents, err := kubeadmconfigutil.MarshalInitConfigurationToBytes(&initCfg, schema.GroupVersion{
+		Group:   "kubeadm.k8s.io",
+		Version: kubeadm.GetKubeadmApisVersion(initConfiguration.KubernetesVersion),
+	})
 	if err != nil {
 		return err
 	}
-
 	if err := ioutil.WriteFile(skuba.KubeadmInitConfFile(), initCfgContents, 0600); err != nil {
 		return errors.Wrap(err, "error writing init configuration")
 	}

pkg/skuba/actions/node/bootstrap/bootstrap.go

@@ -24,8 +24,10 @@ import (
 	"path/filepath"
 
 	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/version"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 
 	"github.com/SUSE/skuba/internal/pkg/skuba/addons"
 	"github.com/SUSE/skuba/internal/pkg/skuba/deployments"
@@ -102,7 +104,10 @@ func coreBootstrap(initConfiguration *kubeadmapi.InitConfiguration, bootstrapCon
 		return errors.Wrap(err, "unable to add target information to init configuration")
 	}
 
-	finalInitConfigurationContents, err := kubeadm.UpdateClusterConfigurationWithClusterVersion(initConfiguration, versionToDeploy)
+	finalInitConfigurationContents, err := kubeadmconfigutil.MarshalInitConfigurationToBytes(initConfiguration, schema.GroupVersion{
+		Group:   "kubeadm.k8s.io",
+		Version: kubeadm.GetKubeadmApisVersion(versionToDeploy),
+	})
 	if err != nil {
 		return errors.Wrap(err, "could not marshal configuration")
 	}
@@ -137,6 +142,7 @@ func coreBootstrap(initConfiguration *kubeadmapi.InitConfiguration, bootstrapCon
 		criSetup,
 		"cri.start",
 		"kubelet.rootcert.upload",
+		"kubelet.servercert.create-and-upload",
 		"kubelet.configure",
 		"kubelet.enable",
 		"kubeadm.init",

pkg/skuba/actions/node/join/join.go

@@ -87,6 +87,7 @@ func Join(client clientset.Interface, joinConfiguration deployments.JoinConfigur
 		criSetup,
 		"cri.start",
 		"kubelet.rootcert.upload",
+		"kubelet.servercert.create-and-upload",
 		"kubelet.configure",
 		"kubelet.enable",
 		"kubeadm.join",

pkg/skuba/actions/node/upgrade/apply.go

@@ -22,8 +22,9 @@ import (
 	"os"
 	"strings"
 
-	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	clientset "k8s.io/client-go/kubernetes"
+	kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 
 	"github.com/SUSE/skuba/internal/pkg/skuba/deployments"
 	"github.com/SUSE/skuba/internal/pkg/skuba/kubeadm"
@@ -32,6 +33,7 @@ import (
 	"github.com/SUSE/skuba/internal/pkg/skuba/node"
 	upgradenode "github.com/SUSE/skuba/internal/pkg/skuba/upgrade/node"
 	"github.com/SUSE/skuba/pkg/skuba"
+	"github.com/pkg/errors"
 )
 
 func Apply(client clientset.Interface, target *deployments.Target) error {
@@ -104,7 +106,11 @@ func Apply(client clientset.Interface, target *deployments.Target) error {
 			initCfg.UseHyperKubeImage = false
 		}
 
-		initCfgContents, err = kubeadm.UpdateClusterConfigurationWithClusterVersion(initCfg, nodeVersionInfoUpdate.Update.APIServerVersion)
+		kubeadm.UpdateClusterConfigurationWithClusterVersion(initCfg, nodeVersionInfoUpdate.Update.APIServerVersion)
+		initCfgContents, err = kubeadmconfigutil.MarshalInitConfigurationToBytes(initCfg, schema.GroupVersion{
+			Group:   "kubeadm.k8s.io",
+			Version: kubeadm.GetKubeadmApisVersion(nodeVersionInfoUpdate.Update.APIServerVersion),
+		})
 		if err != nil {
 			return err
 		}
@@ -177,6 +183,7 @@ func Apply(client clientset.Interface, target *deployments.Target) error {
 	}
 	err = target.Apply(nil,
 		"kubelet.rootcert.upload",
+		"kubelet.servercert.create-and-upload",
 		"kubernetes.restart-services",
 	)
 	if err != nil {