diff --git a/internal/pkg/skuba/deployments/ssh/kubelet.go b/internal/pkg/skuba/deployments/ssh/kubelet.go index aa77f58549..36393e8c62 100644 --- a/internal/pkg/skuba/deployments/ssh/kubelet.go +++ b/internal/pkg/skuba/deployments/ssh/kubelet.go @@ -18,11 +18,17 @@ package ssh import ( + "crypto/x509" "io/ioutil" + "net" "os" "path/filepath" + "strings" + "github.com/pkg/errors" + certutil "k8s.io/client-go/util/cert" "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" "sigs.k8s.io/yaml" "github.com/SUSE/skuba/internal/pkg/skuba/deployments" @@ -33,6 +39,7 @@ import ( func init() { stateMap["kubelet.rootcert.upload"] = kubeletUploadRootCert + stateMap["kubelet.servercert.create-and-upload"] = kubeletCreateAndUploadServerCert stateMap["kubelet.configure"] = kubeletConfigure stateMap["kubelet.enable"] = kubeletEnable } @@ -55,6 +62,83 @@ func kubeletUploadRootCert(t *Target, data interface{}) error { return nil } +func kubeletCreateAndUploadServerCert(t *Target, data interface{}) error { + // Read kubelet root ca certificate and key + caCert, caKey, err := pkiutil.TryLoadCertAndKeyFromDisk(skuba.PkiDir(), kubernetes.KubeletCACertAndKeyBaseName) + if err != nil { + return errors.Wrap(err, "failure loading kubelet CA certificate authority") + } + + host := t.target.Nodename + altNames := certutil.AltNames{} + if ip := net.ParseIP(host); ip != nil { + altNames.IPs = append(altNames.IPs, ip) + } else { + altNames.DNSNames = append(altNames.DNSNames, host) + } + + // Create AltNames with defaults DNSNames/IPs + stdout, _, err := t.silentSsh("hostname", "-I") + if err != nil { + return err + } + for _, addr := range strings.Split(stdout, " ") { + if ip := net.ParseIP(addr); ip != nil { + altNames.IPs = append(altNames.IPs, ip) + } + } + + alternateIPs := []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} + alternateDNS := []string{"localhost"} + if ip := net.ParseIP(t.target.Target); ip != nil { + alternateIPs = append(alternateIPs, ip) + } else { + alternateDNS = append(alternateDNS, t.target.Target) + } + + altNames.IPs = append(altNames.IPs, alternateIPs...) + altNames.DNSNames = append(altNames.DNSNames, alternateDNS...) + + cfg := &pkiutil.CertConfig{ + Config: certutil.Config{ + CommonName: host, + AltNames: altNames, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }, + } + cert, key, err := pkiutil.NewCertAndKey(caCert, caKey, cfg) + if err != nil { + return errors.Wrap(err, "couldn't generate kubelet server certificate") + } + + // Save kubelet server certificate and key to local temporarily + if err := pkiutil.WriteCertAndKey(skuba.PkiDir(), host, cert, key); err != nil { + return errors.Wrapf(err, "failure while saving kubelet server %s certificate and key", host) + } + + // Upload server certificate and key + certPath, keyPath := pkiutil.PathsForCertAndKey(skuba.PkiDir(), host) + if err := t.target.UploadFile(certPath, filepath.Join(kubernetes.KubeletCertAndKeyDir, kubernetes.KubeletServerCertName)); err != nil { + return err + } + if err := t.target.UploadFile(keyPath, filepath.Join(kubernetes.KubeletCertAndKeyDir, kubernetes.KubeletServerKeyName)); err != nil { + return err + } + if _, _, err := t.silentSsh("chmod", "0400", filepath.Join(kubernetes.KubeletCertAndKeyDir, kubernetes.KubeletServerKeyName)); err != nil { + return err + } + + // Remove local temporarily kubelet server certificate and key + if err := os.Remove(certPath); err != nil { + return err + } + if err := os.Remove(keyPath); err != nil { + return err + } + + return nil +} + func kubeletConfigure(t *Target, data interface{}) error { isSUSE, err := t.target.IsSUSEOS() if err != nil { diff --git a/internal/pkg/skuba/kubeadm/configmap.go b/internal/pkg/skuba/kubeadm/configmap.go index 939944188e..fdab4f3e41 100644 --- a/internal/pkg/skuba/kubeadm/configmap.go +++ b/internal/pkg/skuba/kubeadm/configmap.go @@ -25,14 +25,12 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/version" clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" - kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" skubautil "github.com/SUSE/skuba/internal/pkg/skuba/util" ) @@ -132,28 +130,11 @@ func RemoveAPIEndpointFromConfigMap(client clientset.Interface, node *corev1.Nod // UpdateClusterConfigurationWithClusterVersion allows us to set certain configurations during init, but also during upgrades. // The configuration that we put here will be consistently set to newly created configurations, and when we upgrade a cluster. -func UpdateClusterConfigurationWithClusterVersion(initCfg *kubeadmapi.InitConfiguration, clusterVersion *version.Version) ([]byte, error) { +func UpdateClusterConfigurationWithClusterVersion(initCfg *kubeadmapi.InitConfiguration, clusterVersion *version.Version) { // apiserver setApiserverAdmissionPlugins(initCfg, clusterVersion) setContainerImagesWithClusterVersion(initCfg, clusterVersion) setApiserverArgs(initCfg) - - initCfgContents, err := kubeadmconfigutil.MarshalInitConfigurationToBytes(initCfg, schema.GroupVersion{ - Group: "kubeadm.k8s.io", - Version: GetKubeadmApisVersion(clusterVersion), - }) - if err != nil { - return []byte{}, err - } - - // kubelet - kubeletManifest := []byte(`--- -apiVersion: kubelet.config.k8s.io/v1beta1 -kind: KubeletConfiguration -serverTLSBootstrap: true -`) - - return append(initCfgContents, kubeletManifest...), nil } func setApiserverAdmissionPlugins(initCfg *kubeadmapi.InitConfiguration, clusterVersion *version.Version) { diff --git a/internal/pkg/skuba/kubeadm/configmap_test.go b/internal/pkg/skuba/kubeadm/configmap_test.go index 097ea51552..1c0229a2bf 100644 --- a/internal/pkg/skuba/kubeadm/configmap_test.go +++ b/internal/pkg/skuba/kubeadm/configmap_test.go @@ -367,10 +367,7 @@ func TestUpdateClusterConfigurationWithClusterVersion(t *testing.T) { } initCfg.APIServer.ControlPlaneComponent.ExtraArgs["enable-admission-plugins"] = currentAdmissionPlugins } - _, err := UpdateClusterConfigurationWithClusterVersion(&initCfg, tt.clusterVersion) - if err != nil { - t.Errorf("expected no error but an error returned: %v", err) - } + UpdateClusterConfigurationWithClusterVersion(&initCfg, tt.clusterVersion) // Check admission plugins gotAdmissionPlugins := initCfg.APIServer.ControlPlaneComponent.ExtraArgs["enable-admission-plugins"] if gotAdmissionPlugins != expectedAdmissionPlugins { diff --git a/internal/pkg/skuba/kubernetes/kubelet.go b/internal/pkg/skuba/kubernetes/kubelet.go index 8ad0482321..7465abd5a0 100644 --- a/internal/pkg/skuba/kubernetes/kubelet.go +++ b/internal/pkg/skuba/kubernetes/kubelet.go @@ -45,6 +45,13 @@ const ( KubeletCACertName = "kubelet-ca.crt" // KubeletCAKeyName defines kubelet's CA key name KubeletCAKeyName = "kubelet-ca.key" + + // KubeletServerCertAndKeyBaseName defines kubelet server certificate and key base name + KubeletServerCertAndKeyBaseName = "kubelet" + // KubeletServerCertName defines kubelet server certificate name + KubeletServerCertName = "kubelet.crt" + // KubeletServerKeyName defines kubelet server key name + KubeletServerKeyName = "kubelet.key" ) // GenerateKubeletRootCert generates kubelet root CA certificate and key diff --git a/internal/pkg/skuba/kubernetes/versions.go b/internal/pkg/skuba/kubernetes/versions.go index 51d3d80ba0..0516c94497 100644 --- a/internal/pkg/skuba/kubernetes/versions.go +++ b/internal/pkg/skuba/kubernetes/versions.go @@ -132,7 +132,6 @@ var ( Dex: &AddonVersion{"2.16.0", 6}, Gangway: &AddonVersion{"3.1.0-rev4", 5}, MetricsServer: &AddonVersion{"0.3.6", 1}, - Kucero: &AddonVersion{"1.1.1", 0}, PSP: &AddonVersion{"", 4}, }, }, diff --git a/pkg/skuba/actions/cluster/init/init.go b/pkg/skuba/actions/cluster/init/init.go index 17c318ae77..433ece227f 100644 --- a/pkg/skuba/actions/cluster/init/init.go +++ b/pkg/skuba/actions/cluster/init/init.go @@ -33,6 +33,7 @@ import ( kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" + kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "github.com/SUSE/skuba/internal/pkg/skuba/addons" "github.com/SUSE/skuba/internal/pkg/skuba/kubeadm" @@ -283,12 +284,14 @@ func writeKubeadmInitConf(initConfiguration InitConfiguration) error { if len(initConfiguration.CloudProvider) > 0 { updateInitConfigurationWithCloudIntegration(&initCfg, initConfiguration) } - - initCfgContents, err := kubeadm.UpdateClusterConfigurationWithClusterVersion(&initCfg, initConfiguration.KubernetesVersion) + kubeadm.UpdateClusterConfigurationWithClusterVersion(&initCfg, initConfiguration.KubernetesVersion) + initCfgContents, err := kubeadmconfigutil.MarshalInitConfigurationToBytes(&initCfg, schema.GroupVersion{ + Group: "kubeadm.k8s.io", + Version: kubeadm.GetKubeadmApisVersion(initConfiguration.KubernetesVersion), + }) if err != nil { return err } - if err := ioutil.WriteFile(skuba.KubeadmInitConfFile(), initCfgContents, 0600); err != nil { return errors.Wrap(err, "error writing init configuration") } diff --git a/pkg/skuba/actions/node/bootstrap/bootstrap.go b/pkg/skuba/actions/node/bootstrap/bootstrap.go index 93e3dd9608..8e85612a34 100644 --- a/pkg/skuba/actions/node/bootstrap/bootstrap.go +++ b/pkg/skuba/actions/node/bootstrap/bootstrap.go @@ -24,8 +24,10 @@ import ( "path/filepath" "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/version" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "github.com/SUSE/skuba/internal/pkg/skuba/addons" "github.com/SUSE/skuba/internal/pkg/skuba/deployments" @@ -102,7 +104,10 @@ func coreBootstrap(initConfiguration *kubeadmapi.InitConfiguration, bootstrapCon return errors.Wrap(err, "unable to add target information to init configuration") } - finalInitConfigurationContents, err := kubeadm.UpdateClusterConfigurationWithClusterVersion(initConfiguration, versionToDeploy) + finalInitConfigurationContents, err := kubeadmconfigutil.MarshalInitConfigurationToBytes(initConfiguration, schema.GroupVersion{ + Group: "kubeadm.k8s.io", + Version: kubeadm.GetKubeadmApisVersion(versionToDeploy), + }) if err != nil { return errors.Wrap(err, "could not marshal configuration") } @@ -137,6 +142,7 @@ func coreBootstrap(initConfiguration *kubeadmapi.InitConfiguration, bootstrapCon criSetup, "cri.start", "kubelet.rootcert.upload", + "kubelet.servercert.create-and-upload", "kubelet.configure", "kubelet.enable", "kubeadm.init", diff --git a/pkg/skuba/actions/node/join/join.go b/pkg/skuba/actions/node/join/join.go index 4e4c1602a5..21646be743 100644 --- a/pkg/skuba/actions/node/join/join.go +++ b/pkg/skuba/actions/node/join/join.go @@ -87,6 +87,7 @@ func Join(client clientset.Interface, joinConfiguration deployments.JoinConfigur criSetup, "cri.start", "kubelet.rootcert.upload", + "kubelet.servercert.create-and-upload", "kubelet.configure", "kubelet.enable", "kubeadm.join", diff --git a/pkg/skuba/actions/node/upgrade/apply.go b/pkg/skuba/actions/node/upgrade/apply.go index bcdf273eef..5179e6a1a9 100644 --- a/pkg/skuba/actions/node/upgrade/apply.go +++ b/pkg/skuba/actions/node/upgrade/apply.go @@ -22,8 +22,9 @@ import ( "os" "strings" - "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime/schema" clientset "k8s.io/client-go/kubernetes" + kubeadmconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "github.com/SUSE/skuba/internal/pkg/skuba/deployments" "github.com/SUSE/skuba/internal/pkg/skuba/kubeadm" @@ -32,6 +33,7 @@ import ( "github.com/SUSE/skuba/internal/pkg/skuba/node" upgradenode "github.com/SUSE/skuba/internal/pkg/skuba/upgrade/node" "github.com/SUSE/skuba/pkg/skuba" + "github.com/pkg/errors" ) func Apply(client clientset.Interface, target *deployments.Target) error { @@ -104,7 +106,11 @@ func Apply(client clientset.Interface, target *deployments.Target) error { initCfg.UseHyperKubeImage = false } - initCfgContents, err = kubeadm.UpdateClusterConfigurationWithClusterVersion(initCfg, nodeVersionInfoUpdate.Update.APIServerVersion) + kubeadm.UpdateClusterConfigurationWithClusterVersion(initCfg, nodeVersionInfoUpdate.Update.APIServerVersion) + initCfgContents, err = kubeadmconfigutil.MarshalInitConfigurationToBytes(initCfg, schema.GroupVersion{ + Group: "kubeadm.k8s.io", + Version: kubeadm.GetKubeadmApisVersion(nodeVersionInfoUpdate.Update.APIServerVersion), + }) if err != nil { return err } @@ -177,6 +183,7 @@ func Apply(client clientset.Interface, target *deployments.Target) error { } err = target.Apply(nil, "kubelet.rootcert.upload", + "kubelet.servercert.create-and-upload", "kubernetes.restart-services", ) if err != nil {