cri: handle the migration to the new configuration format

cmd/skuba/cluster/upgrade.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/klog"
 
 	"github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
+	"github.com/SUSE/skuba/internal/pkg/skuba/upgrade/cluster"
 	"github.com/SUSE/skuba/pkg/skuba/actions/cluster/upgrade"
 )
 
@@ -37,6 +38,7 @@ func NewUpgradeCmd() *cobra.Command {
 
 	cmd.AddCommand(
 		newUpgradePlanCmd(),
+		newUpgradeLocalConfigCmd(),
 	)
 
 	return cmd
@@ -60,3 +62,17 @@ func newUpgradePlanCmd() *cobra.Command {
 		Args: cobra.NoArgs,
 	}
 }
+
+func newUpgradeLocalConfigCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "localconfig",
+		Short: "Upgrades the local configuration",
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cluster.CriMigrate(); err != nil {
+				klog.Errorf("unable to upgrade the cluster configuration: %s\n", err)
+				os.Exit(1)
+			}
+		},
+		Args: cobra.NoArgs,
+	}
+}

internal/pkg/skuba/deployments/ssh/cri.go

@@ -83,30 +83,9 @@ func criConfigure(t *Target, data interface{}) error {
 	return err
 }
 
+// criSysconfig will enforce the package sysconfig configuration.
 func criSysconfig(t *Target, data interface{}) error {
-	criFiles, err := ioutil.ReadDir(skuba.CriDir())
-	if err != nil {
-		return errors.Wrap(err, "Could not read local cri directory: "+skuba.CriDir())
-	}
-	defer func() {
-		_, _, err := t.ssh("rm -rf /tmp/cri.d")
-		if err != nil {
-			// If the deferred function has any return values, they are discarded when the function completes
-			// https://golang.org/ref/spec#Defer_statements
-			fmt.Println("Could not delete the cri.d config path")
-		}
-	}()
-
-	for _, f := range criFiles {
-		if err := t.target.UploadFile(filepath.Join(skuba.CriDir(), f.Name()), filepath.Join("/tmp/cri.d", f.Name())); err != nil {
-			return err
-		}
-	}
-
-	if _, _, err = t.ssh("mv -f /etc/sysconfig/crio /etc/sysconfig/crio.backup"); err != nil {
-		return err
-	}
-	_, _, err = t.ssh("mv -f /tmp/cri.d/default_flags /etc/sysconfig/crio")
+	_, _, err := t.ssh("cp -f /usr/share/fillup-templates/sysconfig.crio /etc/sysconfig/crio")
 	return err
 }
 

internal/pkg/skuba/upgrade/cluster/cri.go

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2020 SUSE LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cluster
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/SUSE/skuba/internal/pkg/skuba/kubernetes"
+	"github.com/SUSE/skuba/pkg/skuba"
+	clusterinit "github.com/SUSE/skuba/pkg/skuba/actions/cluster/init"
+)
+
+// CriMigrate migrates the old configuration of cri-o < 1.17 to the new format.
+func CriMigrate() error {
+	_, err := os.Stat(skuba.CriDockerDefaultsConfFile())
+	if os.IsNotExist(err) {
+		return nil
+	}
+
+	if err := criGenerateLocalConfiguration(); err != nil {
+		return err
+	}
+
+	if err := criRemoveLocalOldFile(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func criGenerateLocalConfiguration() error {
+	cfg := clusterinit.InitConfiguration{
+		PauseImage:        kubernetes.ComponentContainerImageForClusterVersion(kubernetes.Pause, kubernetes.LatestVersion()),
+		StrictCapDefaults: !criHadStrictCapDefaults(),
+	}
+
+	files := clusterinit.CriScaffoldFiles["criconfig"]
+	for _, file := range files {
+		// Note well: this code will remove the whole local cri configuration
+		// directory if something goes wrong. This is done so to avoid weird
+		// intermediary states in case a file presented a problem. Ideally this
+		// shouldn't happen (it's more of a all or nothing scenario), but take
+		// it into account if you want to reuse this code.
+		if err := clusterinit.WriteScaffoldFile(file, cfg); err != nil {
+			_ = os.RemoveAll(skuba.CriConfDir())
+			return err
+		}
+	}
+	return nil
+}
+
+func criHadStrictCapDefaults() bool {
+	data, err := ioutil.ReadFile(skuba.CriDockerDefaultsConfFile())
+	if err != nil {
+		return false
+	}
+	return strings.Contains(string(data), "--default-capabilities")
+}
+
+func criRemoveLocalOldFile() error {
+	_, err := os.Stat(skuba.CriDockerDefaultsConfFile())
+	if os.IsNotExist(err) {
+		return nil
+	}
+	return os.Remove(skuba.CriDockerDefaultsConfFile())
+}

internal/pkg/skuba/upgrade/cluster/cri_test.go

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2020 SUSE LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cluster
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/SUSE/skuba/pkg/skuba"
+)
+
+func TestCriMigrate(t *testing.T) {
+	// First of all, create the new working directory as it will be taken by
+	// skuba.
+
+	dir, err := ioutil.TempDir("/tmp", "skuba-cri-test")
+	if err != nil {
+		t.Fatalf("Could not initialize test: %v", err)
+	}
+	defer os.RemoveAll(dir)
+	destPath := filepath.Join(dir, "addons", "cri")
+	err = os.MkdirAll(destPath, os.ModePerm)
+	if err != nil {
+		t.Fatalf("Could not initialize test: %v", err)
+	}
+
+	// Move the old `default_flags` we have in `testdata` into the new temporary
+	// working directory.
+
+	b, err := ioutil.ReadFile(filepath.Join("testdata", "addons", "cri", "default_flags"))
+	if err != nil {
+		t.Fatalf("Could not initialize test: %v", err)
+	}
+	err = ioutil.WriteFile(filepath.Join(destPath, "default_flags"), b, 0644)
+	if err != nil {
+		t.Fatalf("Could not initialize test: %v", err)
+	}
+
+	// Now it's safe to change the working directory.
+
+	wd, _ := os.Getwd()
+	defer func() {
+		_ = os.Chdir(wd)
+	}()
+	_ = os.Chdir(dir)
+
+	// Test: before calling the function, check that the expected file exists
+	// there. Then, upon calling the tested function, we should have the new
+	// configuration schema (with `default_capabilities` as given in the old
+	// configuration).
+
+	_, err = os.Stat(skuba.CriDockerDefaultsConfFile())
+	if err != nil {
+		t.Fatalf("File should exist, got '%v' instead", err)
+	}
+
+	err = CriMigrate()
+	if err != nil {
+		t.Fatalf("Function should've run correctly")
+	}
+
+	_, err = os.Stat(skuba.CriDockerDefaultsConfFile())
+	if err == nil || !os.IsNotExist(err) {
+		t.Fatalf("File should not exist, got '%v' instead", err)
+	}
+
+	b, err = ioutil.ReadFile(filepath.Join(destPath, "conf.d", "01-caasp.conf"))
+	if err != nil {
+		t.Fatalf("File should be readable, got '%v' instead", err)
+	}
+	if !strings.Contains(string(b), "default_capabilities") {
+		t.Fatalf("New file should include default capabilities")
+	}
+}

internal/pkg/skuba/upgrade/cluster/testdata/addons/cri/default_flags

@@ -0,0 +1,7 @@
+## Path           : System/Management
+## Description    : Extra cli switches for crio daemon
+## Type           : string
+## Default        : ""
+## ServiceRestart : crio
+#
+CRIO_OPTIONS=--pause-image=registry.suse.com/caasp/v4/pause:3.1 --default-capabilities="CHOWN,DAC_OVERRIDE,FSETID,FOWNER,NET_RAW,SETGID,SETUID,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,MKNOD,AUDIT_WRITE,SETFCAP"

pkg/skuba/actions/cluster/init/constants.go

@@ -28,7 +28,7 @@ type ScaffoldFile struct {
 }
 
 var (
-	criScaffoldFiles = map[string][]ScaffoldFile{
+	CriScaffoldFiles = map[string][]ScaffoldFile{
 		"sysconfig": {
 			{
 				Location: skuba.CriDockerDefaultsConfFile(),

pkg/skuba/actions/cluster/init/init.go

@@ -142,11 +142,7 @@ func isAddonRequired(addon addons.Addon, initConfiguration InitConfiguration) bo
 }
 
 func writeScaffoldFiles(initConfiguration InitConfiguration) error {
-	scaffoldFilesToWrite := criScaffoldFiles["criconfig"]
-	kubernetesVersion := initConfiguration.KubernetesVersion
-	if kubernetesVersion.Minor() < 18 {
-		scaffoldFilesToWrite = criScaffoldFiles["sysconfig"]
-	}
+	scaffoldFilesToWrite := CriScaffoldFiles["criconfig"]
 
 	if len(initConfiguration.CloudProvider) > 0 {
 		if cloudScaffoldFiles, found := cloudScaffoldFiles[initConfiguration.CloudProvider]; found {
@@ -163,31 +159,41 @@ func writeScaffoldFiles(initConfiguration InitConfiguration) error {
 		return errors.Wrapf(err, "could not change to cluster directory %q", initConfiguration.ClusterName)
 	}
 	for _, file := range scaffoldFilesToWrite {
-		filePath, _ := filepath.Split(file.Location)
-		if filePath != "" {
-			if err := os.MkdirAll(filePath, 0700); err != nil {
-				return errors.Wrapf(err, "could not create directory %q", filePath)
-			}
-		}
-		f, err := os.Create(file.Location)
-		if err != nil {
-			return errors.Wrapf(err, "could not create file %q", file.Location)
-		}
-		str, err := renderTemplate(file.Content, initConfiguration)
-		if err != nil {
-			return errors.Wrap(err, "unable to render template")
-		}
-		_, err = f.WriteString(str)
-		if err != nil {
-			return errors.Wrapf(err, "unable to write template to file %s", f.Name())
+		if err := WriteScaffoldFile(file, initConfiguration); err != nil {
+			return err
 		}
-		if err := f.Chmod(0600); err != nil {
-			return errors.Wrapf(err, "unable to chmod file %s", f.Name())
-		}
-		if err := f.Close(); err != nil {
-			return errors.Wrapf(err, "unable to close file %s", f.Name())
+	}
+	return nil
+}
+
+// WriteScaffoldFile writes the given scaffold file into the right location. The
+// initConfiguration is used to render the template itself, since some of the
+// file's contents depend on the configuration.
+func WriteScaffoldFile(file ScaffoldFile, initConfiguration InitConfiguration) error {
+	filePath, _ := filepath.Split(file.Location)
+	if filePath != "" {
+		if err := os.MkdirAll(filePath, 0700); err != nil {
+			return errors.Wrapf(err, "could not create directory %q", filePath)
 		}
 	}
+	f, err := os.Create(file.Location)
+	if err != nil {
+		return errors.Wrapf(err, "could not create file %q", file.Location)
+	}
+	str, err := RenderTemplate(file.Content, initConfiguration)
+	if err != nil {
+		return errors.Wrap(err, "unable to render template")
+	}
+	_, err = f.WriteString(str)
+	if err != nil {
+		return errors.Wrapf(err, "unable to write template to file %s", f.Name())
+	}
+	if err := f.Chmod(0600); err != nil {
+		return errors.Wrapf(err, "unable to chmod file %s", f.Name())
+	}
+	if err := f.Close(); err != nil {
+		return errors.Wrapf(err, "unable to close file %s", f.Name())
+	}
 	return nil
 }
 
@@ -226,7 +232,9 @@ func writeAddonConfigFiles(initConfiguration InitConfiguration) error {
 	return nil
 }
 
-func renderTemplate(templateContents string, initConfiguration InitConfiguration) (string, error) {
+// RenderTemplate renders the given templateContents by using the given
+// initConfiguration.
+func RenderTemplate(templateContents string, initConfiguration InitConfiguration) (string, error) {
 	template, err := template.New("").Parse(templateContents)
 	if err != nil {
 		return "", errors.Wrap(err, "could not parse template")

pkg/skuba/actions/cluster/upgrade/plan.go

@@ -19,6 +19,7 @@ package upgrade
 
 import (
 	"fmt"
+	"os"
 	"sort"
 
 	"github.com/pkg/errors"
@@ -30,6 +31,7 @@ import (
 	skubaconfig "github.com/SUSE/skuba/internal/pkg/skuba/skuba"
 	"github.com/SUSE/skuba/internal/pkg/skuba/upgrade/addon"
 	upgradecluster "github.com/SUSE/skuba/internal/pkg/skuba/upgrade/cluster"
+	"github.com/SUSE/skuba/pkg/skuba"
 )
 
 func Plan(client clientset.Interface) error {
@@ -100,6 +102,8 @@ func plan(client clientset.Interface, availableVersions []*version.Version, clus
 		fmt.Printf("All nodes match the current cluster version: %s.\n", currentClusterVersion.String())
 	}
 
+	checkOldCriFormat()
+
 	planPrePlatformUpgrade(currentClusterVersion, nextClusterVersion, currentAddonVersionInfoUpdate)
 	hasPlatformUpgrade := len(upgradePath) > 0
 	if hasPlatformUpgrade {
@@ -210,3 +214,11 @@ func checkUpdatedAddonsFromClusterVersion(currentClusterVersion *version.Version
 	}
 	return nil
 }
+
+// checkOldCriFormat displays a message if the current cri-o configuration is
+// deemed to be outdated.
+func checkOldCriFormat() {
+	if _, err := os.Stat(skuba.CriDockerDefaultsConfFile()); err == nil {
+		fmt.Printf("\nLocal configuration has to be upgraded: cri-o is using an old format.\n")
+	}
+}