Merge pull request #1690 from martinhoefling/kaniko-host-path-cache

Option to mount HostPath in each Kaniko Pod to be used as cache volume

docs/content/en/schemas/v1beta11.json

@@ -1306,14 +1306,20 @@
     },
     "KanikoCache": {
       "properties": {
+        "hostPath": {
+          "type": "string",
+          "description": "specifies a path on the host that is mounted to each pod as read only cache volume containing base images. If set, must exist on each node and prepopulated with kaniko-warmer.",
+          "x-intellij-html-description": "specifies a path on the host that is mounted to each pod as read only cache volume containing base images. If set, must exist on each node and prepopulated with kaniko-warmer."
+        },
         "repo": {
           "type": "string",
           "description": "a remote repository to store cached layers. If none is specified, one will be inferred from the image name. See [Kaniko Caching](https://github.com/GoogleContainerTools/kaniko#caching).",
           "x-intellij-html-description": "a remote repository to store cached layers. If none is specified, one will be inferred from the image name. See <a href=\"https://github.com/GoogleContainerTools/kaniko#caching\">Kaniko Caching</a>."
         }
       },
       "preferredOrder": [
-        "repo"
+        "repo",
+        "hostPath"
       ],
       "additionalProperties": false,
       "description": "configures Kaniko caching. If a cache is specified, Kaniko will use a remote cache which will speed up builds.",

pkg/skaffold/build/build.go

@@ -31,7 +31,7 @@ type Artifact struct {
 }
 
 // Builder is an interface to the Build API of Skaffold.
-// It must build and make the resulting image accesible to the cluster.
+// It must build and make the resulting image accessible to the cluster.
 // This could include pushing to a authorized repository or loading the nodes with the image.
 // If artifacts is supplied, the builder should only rebuild those artifacts.
 type Builder interface {

pkg/skaffold/build/cluster/kaniko.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/cache"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/cluster/sources"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
@@ -111,6 +112,9 @@ func appendCacheIfExists(args []string, cache *latest.KanikoCache) []string {
 	if cache.Repo != "" {
 		args = append(args, fmt.Sprintf("--cache-repo=%s", cache.Repo))
 	}
+	if cache.HostPath != "" {
+		args = append(args, fmt.Sprintf("--cache-dir=%s", constants.DefaultKanikoDockerConfigPath))
+	}
 	return args
 }
 

pkg/skaffold/build/cluster/sources/gcs.go

@@ -61,7 +61,7 @@ func (g *GCSBucket) Setup(ctx context.Context, out io.Writer, artifact *latest.A
 
 // Pod returns the pod template for this builder
 func (g *GCSBucket) Pod(args []string) *v1.Pod {
-	return podTemplate(g.clusterDetails, g.artifact.Image, args)
+	return podTemplate(g.clusterDetails, g.artifact, args)
 }
 
 // ModifyPod does nothing here, since we just need to let kaniko run to completion

pkg/skaffold/build/cluster/sources/localdir.go

@@ -85,7 +85,7 @@ func (g *LocalDir) Pod(args []string) *v1.Pod {
 		VolumeMounts: []v1.VolumeMount{vm},
 	}
 
-	p := podTemplate(g.clusterDetails, g.artifact.Image, args)
+	p := podTemplate(g.clusterDetails, g.artifact, args)
 	p.Spec.InitContainers = []v1.Container{ic}
 	p.Spec.Containers[0].VolumeMounts = append(p.Spec.Containers[0].VolumeMounts, vm)
 	p.Spec.Volumes = append(p.Spec.Volumes, v)

pkg/skaffold/build/cluster/sources/sources.go

@@ -51,7 +51,7 @@ func Retrieve(clusterDetails *latest.ClusterDetails, artifact *latest.KanikoArti
 	}
 }
 
-func podTemplate(clusterDetails *latest.ClusterDetails, image string, args []string) *v1.Pod {
+func podTemplate(clusterDetails *latest.ClusterDetails, artifact *latest.KanikoArtifact, args []string) *v1.Pod {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "kaniko-",
@@ -62,7 +62,7 @@ func podTemplate(clusterDetails *latest.ClusterDetails, image string, args []str
 			Containers: []v1.Container{
 				{
 					Name:            constants.DefaultKanikoContainerName,
-					Image:           image,
+					Image:           artifact.Image,
 					Args:            args,
 					ImagePullPolicy: v1.PullIfNotPresent,
 					Env: []v1.EnvVar{{
@@ -91,6 +91,25 @@ func podTemplate(clusterDetails *latest.ClusterDetails, image string, args []str
 		},
 	}
 
+	if artifact.Cache != nil && artifact.Cache.HostPath != "" {
+		volumeMount := v1.VolumeMount{
+			Name:      constants.DefaultKanikoCacheDirName,
+			MountPath: constants.DefaultKanikoCacheDirMountPath,
+		}
+
+		pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMount)
+
+		volume := v1.Volume{
+			Name: constants.DefaultKanikoCacheDirName,
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: artifact.Cache.HostPath,
+				},
+			},
+		}
+		pod.Spec.Volumes = append(pod.Spec.Volumes, volume)
+	}
+
 	if clusterDetails.DockerConfig == nil {
 		return pod
 	}

pkg/skaffold/build/cluster/sources/sources_test.go

@@ -196,7 +196,7 @@ func TestPodTemplate(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
-			actual := podTemplate(test.initial, test.image, test.args)
+			actual := podTemplate(test.initial, &latest.KanikoArtifact{Image: test.image, Cache: &latest.KanikoCache{}}, test.args)
 			testutil.CheckDeepEqual(t, test.expected, actual, opt)
 		})
 	}

pkg/skaffold/constants/constants.go

@@ -46,6 +46,8 @@ const (
 	DefaultKanikoContainerName          = "kaniko"
 	DefaultKanikoEmptyDirName           = "kaniko-emptydir"
 	DefaultKanikoEmptyDirMountPath      = "/kaniko/buildcontext"
+	DefaultKanikoCacheDirName           = "kaniko-cache"
+	DefaultKanikoCacheDirMountPath      = "/cache"
 	DefaultKanikoDockerConfigSecretName = "docker-cfg"
 	DefaultKanikoDockerConfigPath       = "/kaniko/.docker"
 

pkg/skaffold/schema/latest/config.go

@@ -221,6 +221,9 @@ type KanikoCache struct {
 	// Repo is a remote repository to store cached layers. If none is specified, one will be
 	// inferred from the image name. See [Kaniko Caching](https://github.com/GoogleContainerTools/kaniko#caching).
 	Repo string `yaml:"repo,omitempty"`
+	// HostPath specifies a path on the host that is mounted to each pod as read only cache volume containing base images.
+	// If set, must exist on each node and prepopulated with kaniko-warmer.
+	HostPath string `yaml:"hostPath,omitempty"`
 }
 
 // ClusterDetails *beta* describes how to do an on-cluster build.