Add cilium helm and kind cluster installation

components/datadog/apps/npm-tools/k8s.go

@@ -3,15 +3,15 @@ package npmtools
 import (
 	"fmt"
 
-	"github.com/DataDog/test-infra-definitions/common/config"
-	"github.com/DataDog/test-infra-definitions/common/utils"
-	componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes"
-
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
 	appsv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1"
 	corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
 	metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
+	"github.com/DataDog/test-infra-definitions/common/config"
+	"github.com/DataDog/test-infra-definitions/common/utils"
+	componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes"
 )
 
 func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace string, testURL string, opts ...pulumi.ResourceOption) (*componentskube.Workload, error) {

components/kubernetes/cilium/cilium.go

@@ -0,0 +1,62 @@
+package cilium
+
+import (
+	"reflect"
+
+	kubeHelm "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
+	"github.com/DataDog/test-infra-definitions/common"
+)
+
+type Params struct {
+	HelmValues HelmValues
+	Version    string
+}
+
+type Option = func(*Params) error
+
+func NewParams(options ...Option) (*Params, error) {
+	return common.ApplyOption(&Params{}, options)
+}
+
+func WithHelmValues(values HelmValues) Option {
+	return func(p *Params) error {
+		p.HelmValues = values
+		return nil
+	}
+}
+
+func WithVersion(version string) Option {
+	return func(p *Params) error {
+		p.Version = version
+		return nil
+	}
+}
+
+type HelmComponent struct {
+	pulumi.ResourceState
+
+	CiliumHelmReleaseStatus kubeHelm.ReleaseStatusOutput
+}
+
+func boolValue(i pulumi.Input) bool {
+	pv := reflect.ValueOf(i)
+	if pv.Kind() == reflect.Ptr {
+		if pv.IsNil() {
+			return false
+		}
+
+		pv = pv.Elem()
+	}
+
+	return pv.Bool()
+}
+
+func (p *Params) hasKubeProxyReplacement() bool {
+	if v, ok := p.HelmValues["kubeProxyReplacement"]; ok {
+		return boolValue(v)
+	}
+
+	return false
+}

components/kubernetes/cilium/helm.go

@@ -0,0 +1,47 @@
+package cilium
+
+import (
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
+	"github.com/DataDog/test-infra-definitions/common/config"
+	"github.com/DataDog/test-infra-definitions/components/kubernetes"
+	"github.com/DataDog/test-infra-definitions/resources/helm"
+)
+
+type HelmValues pulumi.Map
+
+func NewHelmInstallation(e config.Env, cluster *kubernetes.Cluster, params *Params, opts ...pulumi.ResourceOption) (*HelmComponent, error) {
+	helmComponent := &HelmComponent{}
+	if err := e.Ctx().RegisterComponentResource("dd:cilium", "cilium", helmComponent, opts...); err != nil {
+		return nil, err
+	}
+
+	if params.hasKubeProxyReplacement() {
+		params.HelmValues["k8sServiceHost"] = cluster.KubeInternalServerAddress
+		params.HelmValues["k8sServicePort"] = cluster.KubeInternalServerPort
+	}
+
+	opts = append(opts, pulumi.Parent(helmComponent))
+	ciliumBase, err := helm.NewInstallation(e, helm.InstallArgs{
+		RepoURL:     "https://helm.cilium.io",
+		ChartName:   "cilium",
+		InstallName: "cilium",
+		Namespace:   "kube-system",
+		Values:      pulumi.Map(params.HelmValues),
+		Version:     pulumi.StringPtr(params.Version),
+	}, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	helmComponent.CiliumHelmReleaseStatus = ciliumBase.Status
+	resourceOutputs := pulumi.Map{
+		"CiliumBaseHelmReleaseStatus": ciliumBase.Status,
+	}
+
+	if err := e.Ctx().RegisterResourceOutputs(helmComponent, resourceOutputs); err != nil {
+		return nil, err
+	}
+
+	return helmComponent, nil
+}

components/kubernetes/cilium/kind-cilium-cluster.yaml

@@ -0,0 +1,22 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  extraMounts:
+  - hostPath: /proc
+    containerPath: /host/proc
+- role: worker
+  extraMounts:
+  - hostPath: /proc
+    containerPath: /host/proc
+containerdConfigPatches:
+  - |-
+    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+      endpoint = ["https://mirror.gcr.io", "https://registry-1.docker.io"]
+networking:
+  apiServerAddress: "0.0.0.0"
+  apiServerPort: 8443
+  disableDefaultCNI: true
+{{if .KubeProxyReplacement}}
+  kubeProxyMode: "none"
+{{end}}

components/kubernetes/cilium/kind.go

@@ -0,0 +1,97 @@
+package cilium
+
+import (
+	"embed"
+	"fmt"
+	"net/url"
+	"strings"
+	"text/template"
+
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"gopkg.in/yaml.v3"
+
+	"github.com/DataDog/test-infra-definitions/common/config"
+	"github.com/DataDog/test-infra-definitions/common/utils"
+	"github.com/DataDog/test-infra-definitions/components/command"
+	"github.com/DataDog/test-infra-definitions/components/kubernetes"
+	"github.com/DataDog/test-infra-definitions/components/remote"
+)
+
+//go:embed kind-cilium-cluster.yaml
+var kindCilumClusterFS embed.FS
+
+func kindKubeClusterConfigFromCiliumParams(params *Params) (string, error) {
+	o := struct {
+		KubeProxyReplacement bool
+	}{
+		KubeProxyReplacement: params.hasKubeProxyReplacement(),
+	}
+
+	kindCiliumClusterTemplate, err := template.ParseFS(kindCilumClusterFS, "kind-cilium-cluster.yaml")
+	if err != nil {
+		return "", err
+	}
+
+	var kindCilumClusterConfig strings.Builder
+	if err = kindCiliumClusterTemplate.Execute(&kindCilumClusterConfig, o); err != nil {
+		return "", err
+	}
+
+	return kindCilumClusterConfig.String(), nil
+}
+
+func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion string, ciliumOpts []Option, opts ...pulumi.ResourceOption) (*kubernetes.Cluster, error) {
+	params, err := NewParams(ciliumOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("could not create cilium params from opts: %w", err)
+	}
+
+	clusterConfig, err := kindKubeClusterConfigFromCiliumParams(params)
+	if err != nil {
+		return nil, err
+	}
+
+	cluster, err := kubernetes.NewKindClusterWithConfig(env, vm, name, kubeVersion, clusterConfig, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	if params.hasKubeProxyReplacement() {
+		runner := vm.OS.Runner()
+		kindClusterName := env.CommonNamer().DisplayName(49) // We can have some issues if the name is longer than 50 characters
+		kubeConfigInternalCmd, err := runner.Command(
+			env.CommonNamer().ResourceName("kube-kubeconfig-internal"),
+			&command.Args{
+				Create: pulumi.Sprintf("kind get kubeconfig --name %s --internal", kindClusterName),
+			},
+			utils.MergeOptions(opts, utils.PulumiDependsOn(cluster))...,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		hostPort := kubeConfigInternalCmd.StdoutOutput().ApplyT(
+			func(v string) ([]string, error) {
+				out := map[string]interface{}{}
+				if err := yaml.Unmarshal([]byte(v), out); err != nil {
+					return nil, fmt.Errorf("error unmarshaling output of kubeconfig: %w", err)
+				}
+
+				clusters := out["clusters"].([]interface{})
+				cluster := clusters[0].(map[string]interface{})["cluster"]
+				server := cluster.(map[string]interface{})["server"].(string)
+				u, err := url.Parse(server)
+				if err != nil {
+					return nil, fmt.Errorf("could not parse server address %s: %w", server, err)
+				}
+
+				return []string{u.Hostname(), u.Port()}, nil
+			},
+		).(pulumi.StringArrayOutput)
+
+		cluster.KubeInternalServerAddress = hostPort.Index(pulumi.Int(0))
+		cluster.KubeInternalServerPort = hostPort.Index(pulumi.Int(1))
+	}
+
+	return cluster, nil
+}

components/kubernetes/component.go

@@ -1,9 +1,10 @@
 package kubernetes
 
 import (
-	"github.com/DataDog/test-infra-definitions/components"
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
+	"github.com/DataDog/test-infra-definitions/components"
 )
 
 // The type that is used to import the KubernetesCluster component
@@ -21,8 +22,10 @@ type Cluster struct {
 
 	KubeProvider *kubernetes.Provider
 
-	ClusterName pulumi.StringOutput `pulumi:"clusterName"`
-	KubeConfig  pulumi.StringOutput `pulumi:"kubeConfig"`
+	ClusterName               pulumi.StringOutput `pulumi:"clusterName"`
+	KubeConfig                pulumi.StringOutput `pulumi:"kubeConfig"`
+	KubeInternalServerAddress pulumi.StringOutput `pulumi:"kubeInternalServerAddress"`
+	KubeInternalServerPort    pulumi.StringOutput `pulumi:"kubeInternalServerPort"`
 }
 
 func (c *Cluster) Export(ctx *pulumi.Context, out *ClusterOutput) error {

components/kubernetes/istio/istio.go

@@ -119,7 +119,7 @@ func NewHelmInstallation(e config.Env, opts ...pulumi.ResourceOption) (*HelmComp
 	return helmComponent, nil
 }
 
-func NewHttpbinServiceInstallation(e config.CommonEnvironment, opts ...pulumi.ResourceOption) (*corev1.Service, error) {
+func NewHttpbinServiceInstallation(e config.Env, opts ...pulumi.ResourceOption) (*corev1.Service, error) {
 	// deploy httpbin on default namespace
 	httpbinServiceAccount, err := corev1.NewServiceAccount(e.Ctx(), "httpbin", &corev1.ServiceAccountArgs{
 		Metadata: metav1.ObjectMetaArgs{Name: pulumi.String("httpbin")},

components/kubernetes/k8sapply/component.go

@@ -2,6 +2,7 @@ package k8sapply
 
 import (
 	"fmt"
+
 	"github.com/DataDog/test-infra-definitions/common/config"
 	kubeComp "github.com/DataDog/test-infra-definitions/components/kubernetes"
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"

components/kubernetes/kind.go

@@ -3,19 +3,18 @@ package kubernetes
 import (
 	_ "embed"
 	"fmt"
-
 	"regexp"
 	"strings"
 
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+
 	"github.com/DataDog/test-infra-definitions/common/config"
 	"github.com/DataDog/test-infra-definitions/common/utils"
 	"github.com/DataDog/test-infra-definitions/components"
 	"github.com/DataDog/test-infra-definitions/components/command"
 	"github.com/DataDog/test-infra-definitions/components/docker"
 	"github.com/DataDog/test-infra-definitions/components/os"
 	"github.com/DataDog/test-infra-definitions/components/remote"
-
-	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 
 const (
@@ -28,6 +27,10 @@ var kindClusterConfig string
 
 // Install Kind on a Linux virtual machine.
 func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion string, opts ...pulumi.ResourceOption) (*Cluster, error) {
+	return NewKindClusterWithConfig(env, vm, name, kubeVersion, kindClusterConfig, opts...)
+}
+
+func NewKindClusterWithConfig(env config.Env, vm *remote.Host, name string, kubeVersion, kindConfig string, opts ...pulumi.ResourceOption) (*Cluster, error) {
 	return components.NewComponent(env, name, func(clusterComp *Cluster) error {
 		kindClusterName := env.CommonNamer().DisplayName(49) // We can have some issues if the name is longer than 50 characters
 		opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp))
@@ -57,7 +60,7 @@ func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion st
 
 		clusterConfigFilePath := fmt.Sprintf("/tmp/kind-cluster-%s.yaml", name)
 		clusterConfig, err := vm.OS.FileManager().CopyInlineFile(
-			pulumi.String(kindClusterConfig),
+			pulumi.String(kindConfig),
 			clusterConfigFilePath, opts...)
 		if err != nil {
 			return err
@@ -69,7 +72,7 @@ func NewKindCluster(env config.Env, vm *remote.Host, name string, kubeVersion st
 			&command.Args{
 				Create:   pulumi.Sprintf("kind create cluster --name %s --config %s --image %s --wait %s", kindClusterName, clusterConfigFilePath, nodeImage, kindReadinessWait),
 				Delete:   pulumi.Sprintf("kind delete cluster --name %s", kindClusterName),
-				Triggers: pulumi.Array{pulumi.String(kindClusterConfig)},
+				Triggers: pulumi.Array{pulumi.String(kindConfig)},
 			},
 			utils.MergeOptions(opts, utils.PulumiDependsOn(clusterConfig, kindInstall), pulumi.DeleteBeforeReplace(true))...,
 		)

resources/helm/helm.go

@@ -14,6 +14,7 @@ type InstallArgs struct {
 	Namespace   string
 	ValuesYAML  pulumi.AssetOrArchiveArrayInput
 	Values      pulumi.MapInput
+	Version     pulumi.StringPtrInput
 }
 
 // Important: set relevant Kubernetes provider in `opts`
@@ -29,5 +30,6 @@ func NewInstallation(e config.Env, args InstallArgs, opts ...pulumi.ResourceOpti
 		DependencyUpdate: pulumi.BoolPtr(true),
 		ValueYamlFiles:   args.ValuesYAML,
 		Values:           args.Values,
+		Version:          args.Version,
 	}, opts...)
 }