Add node affinity to kit substrate components (#178)

* Add node affinity to kit substrate components * Implement review comments
awslabs · Apr 13, 2022 · 97715e3 · 97715e3
1 parent 1c951ea
commit 97715e3
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 7 deletions.
diff --git a/operator/charts/kit-operator/templates/controller/deployment.yaml b/operator/charts/kit-operator/templates/controller/deployment.yaml
@@ -25,4 +25,16 @@ spec:
             memory: 20Mi
       # https://github.com/aws/amazon-eks-pod-identity-webhook/issues/8#issuecomment-636888074
       securityContext:
-        fsGroup: 1000
+        fsGroup: 1000
+      {{- with .Values.controller.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
diff --git a/operator/charts/kit-operator/templates/webhook/deployment.yaml b/operator/charts/kit-operator/templates/webhook/deployment.yaml
@@ -43,4 +43,17 @@ spec:
                   fieldPath: metadata.namespace
       # https://github.com/aws/amazon-eks-pod-identity-webhook/issues/8#issuecomment-636888074
       securityContext:
-        fsGroup: 1000
+        fsGroup: 1000
+      {{- with .Values.webhook.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.webhook.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.webhook.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+
diff --git a/operator/charts/kit-operator/values.yaml b/operator/charts/kit-operator/values.yaml
@@ -17,4 +17,4 @@ webhook:
   tolerations: []
   affinity: {}
   # TODO this will be updated by the git actions
-  image: "public.ecr.aws/kit/kit-webhook:latest"
+  image: "public.ecr.aws/kit/kit-webhook:latest"
diff --git a/substrate/pkg/controller/substrate/cluster/addons/kitoperator.go b/substrate/pkg/controller/substrate/cluster/addons/kitoperator.go
@@ -33,9 +33,13 @@ func (l *KITOperator) Create(ctx context.Context, substrate *v1alpha1.Substrate)
 	if err := helm.NewClient(*substrate.Status.Cluster.KubeConfig).Apply(ctx, &helm.Chart{
 		Namespace:       "kit",
 		Name:            "kit-operator",
-		Repository:      "https://github.com/awslabs/kubernetes-iteration-toolkit/releases/download/kit-operator-0.0.12",
-		Version:         "0.0.12",
+		Repository:      "https://github.com/awslabs/kubernetes-iteration-toolkit/releases/download/latest",
+		Version:         "latest",
 		CreateNamespace: true,
+		Values: map[string]interface{}{
+			"controller": map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
+			"webhook":    map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
+		},
 	}); err != nil {
 		return reconcile.Result{}, fmt.Errorf("applying chart, %w", err)
 	}

diff --git a/substrate/pkg/controller/substrate/cluster/addons/prometheusstack.go b/substrate/pkg/controller/substrate/cluster/addons/prometheusstack.go
@@ -44,8 +44,9 @@ func (p *PrometheusStack) Create(ctx context.Context, substrate *v1alpha1.Substr
 			"kubeApiServer":         map[string]interface{}{"enabled": false},
 			"kubeStateMetrics":      map[string]interface{}{"enabled": false},
 			"kubeControllerManager": map[string]interface{}{"enabled": false},
-			"prometheus":            map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}},
-			"prometheusOperator":    map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}},
+			"prometheus":            map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}, "prometheusSpec": map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}}},
+			"prometheusOperator":    map[string]interface{}{"serviceMonitor": map[string]interface{}{"selfMonitor": false}, "nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
+			"grafana":               map[string]interface{}{"nodeSelector": map[string]interface{}{"kit.aws/substrate": "control-plane"}},
 		},
 	}); err != nil {
 		return reconcile.Result{}, fmt.Errorf("applying chart, %w", err)