Skip to content

Commit 236126b

Browse files
jabellardjabellard
authored
Automated cherry pick of #6180: Fix issues with running multiple Karmada instances in the (#6200)
* Fix issues with running multiple Karmada instances in the same namespace Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> * Fix issues with running multiple Karmada instances in the same namespace Signed-off-by: Joe Nathan Abellard <contact@jabellard.com> --------- Signed-off-by: Joe Nathan Abellard <contact@jabellard.com>
1 parent ecc12b4 commit 236126b

22 files changed

+514
-438
lines changed

‎hack/deploy-karmada-operator.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,4 @@ kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REP
7070
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" apply -f "${REPO_ROOT}/operator/config/deploy/karmada-operator-deployment.yaml"
7171

7272
# wait karmada-operator ready
73-
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=30s pods -l karmada-app=karmada-operator -n ${KARMADA_SYSTEM_NAMESPACE}
73+
kubectl --kubeconfig="${KUBECONFIG}" --context="${CONTEXT_NAME}" wait --for=condition=Ready --timeout=30s pods -l app.kubernetes.io/name=karmada-operator -n ${KARMADA_SYSTEM_NAMESPACE}

‎hack/local-up-karmada-by-operator.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ cd -
8484
CRDTARBALL_URL="http://local"
8585
DATA_DIR="/var/lib/karmada"
8686
CRD_CACHE_DIR=$(getCrdsDir "${DATA_DIR}" "${CRDTARBALL_URL}")
87-
OPERATOR_POD_NAME=$(kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" get pods -n ${KARMADA_SYSTEM_NAMESPACE} -l karmada-app=karmada-operator -o custom-columns=NAME:.metadata.name --no-headers)
87+
OPERATOR_POD_NAME=$(kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" get pods -n ${KARMADA_SYSTEM_NAMESPACE} -l app.kubernetes.io/name=karmada-operator -o custom-columns=NAME:.metadata.name --no-headers)
8888
kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" exec -i ${OPERATOR_POD_NAME} -n ${KARMADA_SYSTEM_NAMESPACE} -- mkdir -p ${CRD_CACHE_DIR}
8989
kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" cp ${REPO_ROOT}/crds.tar.gz ${KARMADA_SYSTEM_NAMESPACE}/${OPERATOR_POD_NAME}:${CRD_CACHE_DIR}
9090

‎hack/operator-e2e-environment.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,6 @@ cd -
7777
CRDTARBALL_URL="http://local"
7878
DATA_DIR="/var/lib/karmada"
7979
CRD_CACHE_DIR=$(getCrdsDir "${DATA_DIR}" "${CRDTARBALL_URL}")
80-
OPERATOR_POD_NAME=$(kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" get pods -n ${KARMADA_SYSTEM_NAMESPACE} -l karmada-app=karmada-operator -o custom-columns=NAME:.metadata.name --no-headers)
80+
OPERATOR_POD_NAME=$(kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" get pods -n ${KARMADA_SYSTEM_NAMESPACE} -l app.kubernetes.io/name=karmada-operator -o custom-columns=NAME:.metadata.name --no-headers)
8181
kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" exec -i ${OPERATOR_POD_NAME} -n ${KARMADA_SYSTEM_NAMESPACE} -- mkdir -p ${CRD_CACHE_DIR}
8282
kubectl --kubeconfig="${MAIN_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" cp ${REPO_ROOT}/crds.tar.gz ${KARMADA_SYSTEM_NAMESPACE}/${OPERATOR_POD_NAME}:${CRD_CACHE_DIR}

‎operator/config/deploy/karmada-operator-clusterrole.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ kind: ClusterRole
33
metadata:
44
name: karmada-operator
55
labels:
6-
karmada-app: karmada-operator
6+
app.kubernetes.io/name: karmada-operator
77
rules:
88
- apiGroups: ["coordination.k8s.io"]
99
resources: ["leases"] # karmada-operator requires access to the Lease resource for leader election

‎operator/config/deploy/karmada-operator-clusterrolebinding.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ kind: ClusterRoleBinding
33
metadata:
44
name: karmada-operator
55
labels:
6-
karmada-app: karmada-operator
6+
app.kubernetes.io/name: karmada-operator
77
roleRef:
88
apiGroup: rbac.authorization.k8s.io
99
kind: ClusterRole

‎operator/config/deploy/karmada-operator-deployment.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ metadata:
44
name: karmada-operator
55
namespace: karmada-system
66
labels:
7-
karmada-app: karmada-operator
7+
app.kubernetes.io/name: karmada-operator
88
spec:
99
replicas: 1
1010
selector:
1111
matchLabels:
12-
karmada-app: karmada-operator
12+
app.kubernetes.io/name: karmada-operator
1313
template:
1414
metadata:
1515
labels:
16-
karmada-app: karmada-operator
16+
app.kubernetes.io/name: karmada-operator
1717
spec:
1818
containers:
1919
- name: karmada-operator

‎operator/config/deploy/karmada-operator-serviceaccount.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ metadata:
44
name: karmada-operator
55
namespace: karmada-system
66
labels:
7-
karmada-app: karmada-operator
7+
app.kubernetes.io/name: karmada-operator

‎operator/pkg/constants/constants.go

+5
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,11 @@ const (
121121

122122
// APIServiceName defines the karmada aggregated apiserver APIService resource name.
123123
APIServiceName = "v1alpha1.cluster.karmada.io"
124+
125+
// AppNameLabel defines the recommended label for identifying an application.
126+
AppNameLabel = "app.kubernetes.io/name"
127+
// AppInstanceLabel defines the recommended label for identifying an application instance.
128+
AppInstanceLabel = "app.kubernetes.io/instance"
124129
)
125130

126131
var (

‎operator/pkg/controlplane/apiserver/apiserver.go

+31-27
Original file line numberDiff line numberDiff line change
@@ -52,17 +52,18 @@ func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv
5252

5353
func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, _ map[string]bool) error {
5454
apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct {
55-
DeploymentName, Namespace, Image, ImagePullPolicy string
56-
ServiceSubnet, KarmadaCertsSecret string
57-
Replicas *int32
55+
KarmadaInstanceName, DeploymentName, Namespace, Image, ImagePullPolicy string
56+
ServiceSubnet, KarmadaCertsSecret string
57+
Replicas *int32
5858
}{
59-
DeploymentName: util.KarmadaAPIServerName(name),
60-
Namespace: namespace,
61-
Image: cfg.Image.Name(),
62-
ImagePullPolicy: string(cfg.ImagePullPolicy),
63-
ServiceSubnet: *cfg.ServiceSubnet,
64-
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
65-
Replicas: cfg.Replicas,
59+
KarmadaInstanceName: name,
60+
DeploymentName: util.KarmadaAPIServerName(name),
61+
Namespace: namespace,
62+
Image: cfg.Image.Name(),
63+
ImagePullPolicy: string(cfg.ImagePullPolicy),
64+
ServiceSubnet: *cfg.ServiceSubnet,
65+
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
66+
Replicas: cfg.Replicas,
6667
})
6768
if err != nil {
6869
return fmt.Errorf("error when parsing karmadaApiserver deployment template: %w", err)
@@ -91,11 +92,12 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
9192

9293
func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string) error {
9394
karmadaApiserverServiceBytes, err := util.ParseTemplate(KarmadaApiserverService, struct {
94-
ServiceName, Namespace, ServiceType string
95+
KarmadaInstanceName, ServiceName, Namespace, ServiceType string
9596
}{
96-
ServiceName: util.KarmadaAPIServerName(name),
97-
Namespace: namespace,
98-
ServiceType: string(cfg.ServiceType),
97+
KarmadaInstanceName: name,
98+
ServiceName: util.KarmadaAPIServerName(name),
99+
Namespace: namespace,
100+
ServiceType: string(cfg.ServiceType),
99101
})
100102
if err != nil {
101103
return fmt.Errorf("error when parsing karmadaApiserver serive template: %w", err)
@@ -117,17 +119,18 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al
117119

118120
func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error {
119121
aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct {
120-
DeploymentName, Namespace, Image, ImagePullPolicy string
121-
KubeconfigSecret, KarmadaCertsSecret string
122-
Replicas *int32
122+
KarmadaInstanceName, DeploymentName, Namespace, Image, ImagePullPolicy string
123+
KubeconfigSecret, KarmadaCertsSecret string
124+
Replicas *int32
123125
}{
124-
DeploymentName: util.KarmadaAggregatedAPIServerName(name),
125-
Namespace: namespace,
126-
Image: cfg.Image.Name(),
127-
ImagePullPolicy: string(cfg.ImagePullPolicy),
128-
KubeconfigSecret: util.ComponentKarmadaConfigSecretName(util.KarmadaAggregatedAPIServerName(name)),
129-
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
130-
Replicas: cfg.Replicas,
126+
KarmadaInstanceName: name,
127+
DeploymentName: util.KarmadaAggregatedAPIServerName(name),
128+
Namespace: namespace,
129+
Image: cfg.Image.Name(),
130+
ImagePullPolicy: string(cfg.ImagePullPolicy),
131+
KubeconfigSecret: util.ComponentKarmadaConfigSecretName(util.KarmadaAggregatedAPIServerName(name)),
132+
KarmadaCertsSecret: util.KarmadaCertSecretName(name),
133+
Replicas: cfg.Replicas,
131134
})
132135
if err != nil {
133136
return fmt.Errorf("error when parsing karmadaAggregatedAPIServer deployment template: %w", err)
@@ -155,10 +158,11 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
155158

156159
func createKarmadaAggregatedAPIServerService(client clientset.Interface, name, namespace string) error {
157160
aggregatedAPIServerServiceBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerService, struct {
158-
ServiceName, Namespace string
161+
KarmadaInstanceName, ServiceName, Namespace string
159162
}{
160-
ServiceName: util.KarmadaAggregatedAPIServerName(name),
161-
Namespace: namespace,
163+
KarmadaInstanceName: name,
164+
ServiceName: util.KarmadaAggregatedAPIServerName(name),
165+
Namespace: namespace,
162166
})
163167
if err != nil {
164168
return fmt.Errorf("error when parsing karmadaAggregatedAPIServer serive template: %w", err)

‎operator/pkg/controlplane/apiserver/manifests.go

+31-17
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,22 @@ apiVersion: apps/v1
2323
kind: Deployment
2424
metadata:
2525
labels:
26-
karmada-app: karmada-apiserver
2726
app.kubernetes.io/managed-by: karmada-operator
27+
app.kubernetes.io/name: karmada-apiserver
28+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
2829
name: {{ .DeploymentName }}
2930
namespace: {{ .Namespace }}
3031
spec:
3132
replicas: {{ .Replicas }}
3233
selector:
3334
matchLabels:
34-
karmada-app: karmada-apiserver
35+
app.kubernetes.io/name: karmada-apiserver
36+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
3537
template:
3638
metadata:
3739
labels:
38-
karmada-app: karmada-apiserver
40+
app.kubernetes.io/name: karmada-apiserver
41+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
3942
spec:
4043
automountServiceAccountToken: false
4144
containers:
@@ -69,7 +72,6 @@ spec:
6972
- --max-requests-inflight=1500
7073
- --max-mutating-requests-inflight=500
7174
- --v=4
72-
7375
livenessProbe:
7476
failureThreshold: 8
7577
httpGet:
@@ -94,11 +96,15 @@ spec:
9496
podAntiAffinity:
9597
requiredDuringSchedulingIgnoredDuringExecution:
9698
- labelSelector:
97-
matchExpressions:
98-
- key: karmada-app
99-
operator: In
100-
values:
101-
- karmada-apiserver
99+
matchExpressions:
100+
- key: app.kubernetes.io/name
101+
operator: In
102+
values:
103+
- karmada-apiserver
104+
- key: app.kubernetes.io/instance
105+
operator: In
106+
values:
107+
- {{ .KarmadaInstanceName }}
102108
topologyKey: kubernetes.io/hostname
103109
ports:
104110
- containerPort: 5443
@@ -120,8 +126,9 @@ apiVersion: v1
120126
kind: Service
121127
metadata:
122128
labels:
123-
karmada-app: karmada-apiserver
124129
app.kubernetes.io/managed-by: karmada-operator
130+
app.kubernetes.io/name: karmada-apiserver
131+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
125132
name: {{ .ServiceName }}
126133
namespace: {{ .Namespace }}
127134
spec:
@@ -131,29 +138,33 @@ spec:
131138
protocol: TCP
132139
targetPort: 5443
133140
selector:
134-
karmada-app: karmada-apiserver
141+
app.kubernetes.io/name: karmada-apiserver
142+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
135143
type: {{ .ServiceType }}
136144
`
137145

138-
// KarmadaAggregatedAPIServerDeployment is karmada aggreagated apiserver deployment manifest
146+
// KarmadaAggregatedAPIServerDeployment is karmada aggregated apiserver deployment manifest
139147
KarmadaAggregatedAPIServerDeployment = `
140148
apiVersion: apps/v1
141149
kind: Deployment
142150
metadata:
143151
labels:
144-
karmada-app: karmada-aggregated-apiserver
145152
app.kubernetes.io/managed-by: karmada-operator
153+
app.kubernetes.io/name: karmada-aggregated-apiserver
154+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
146155
name: {{ .DeploymentName }}
147156
namespace: {{ .Namespace }}
148157
spec:
149158
replicas: {{ .Replicas }}
150159
selector:
151160
matchLabels:
152-
karmada-app: karmada-aggregated-apiserver
161+
app.kubernetes.io/name: karmada-aggregated-apiserver
162+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
153163
template:
154164
metadata:
155165
labels:
156-
karmada-app: karmada-aggregated-apiserver
166+
app.kubernetes.io/name: karmada-aggregated-apiserver
167+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
157168
spec:
158169
automountServiceAccountToken: false
159170
containers:
@@ -185,14 +196,16 @@ spec:
185196
secret:
186197
secretName: {{ .KarmadaCertsSecret }}
187198
`
199+
188200
// KarmadaAggregatedAPIServerService is karmada aggregated APIServer Service manifest
189201
KarmadaAggregatedAPIServerService = `
190202
apiVersion: v1
191203
kind: Service
192204
metadata:
193205
labels:
194-
karmada-app: karmada-aggregated-apiserver
195206
app.kubernetes.io/managed-by: karmada-operator
207+
app.kubernetes.io/name: karmada-aggregated-apiserver
208+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
196209
name: {{ .ServiceName }}
197210
namespace: {{ .Namespace }}
198211
spec:
@@ -201,7 +214,8 @@ spec:
201214
protocol: TCP
202215
targetPort: 443
203216
selector:
204-
karmada-app: karmada-aggregated-apiserver
217+
app.kubernetes.io/name: karmada-aggregated-apiserver
218+
app.kubernetes.io/instance: {{ .KarmadaInstanceName }}
205219
type: ClusterIP
206220
`
207221
)

0 commit comments

Comments
 (0)