From 13d53aa45d1c22c22a20bd293d45365340b9edc5 Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Tue, 6 Aug 2024 22:31:22 -0500 Subject: [PATCH 1/3] Add hetzner server labels for IG node labels --- pkg/model/context.go | 48 ++++++++++++++++++---------- pkg/model/hetznermodel/servers.go | 9 +++--- pkg/nodeidentity/hetzner/identify.go | 5 ++- upup/pkg/fi/cloudup/hetzner/cloud.go | 1 + 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/pkg/model/context.go b/pkg/model/context.go index 7e3e7ea1c520f..0dd47fdbe4a30 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -33,6 +33,7 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "k8s.io/kops/upup/pkg/fi/cloudup/hetzner" "k8s.io/kops/upup/pkg/fi/cloudup/scaleway" "github.com/blang/semver/v4" @@ -174,7 +175,12 @@ func (b *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma return nil, fmt.Errorf("error building node labels: %w", err) } for k, v := range nodeLabels { - labels[nodeidentityaws.ClusterAutoscalerNodeTemplateLabel+k] = v + switch b.Cluster.GetCloudProvider() { + case kops.CloudProviderHetzner: + labels[hetzner.TagKubernetesNodeLabelPrefix+k] = v + default: + labels[nodeidentityaws.ClusterAutoscalerNodeTemplateLabel+k] = v + } } // Apply labels for cluster autoscaler node taints @@ -185,27 +191,33 @@ func (b *KopsModelContext) CloudTagsForInstanceGroup(ig *kops.InstanceGroup) (ma } } - // The system tags take priority because the cluster likely breaks without them... + switch b.Cluster.GetCloudProvider() { + case kops.CloudProviderHetzner: + labels[hetzner.TagKubernetesInstanceRole] = string(ig.Spec.Role) + labels[hetzner.TagKubernetesClusterName] = b.ClusterName() + labels[hetzner.TagKubernetesInstanceGroup] = ig.Name + default: + // The system tags take priority because the cluster likely breaks without them... + + if ig.Spec.Role == kops.InstanceGroupRoleControlPlane { + labels[awstasks.CloudTagInstanceGroupRolePrefix+"master"] = "1" + labels[awstasks.CloudTagInstanceGroupRolePrefix+kops.InstanceGroupRoleControlPlane.ToLowerString()] = "1" + } - if ig.Spec.Role == kops.InstanceGroupRoleControlPlane { - labels[awstasks.CloudTagInstanceGroupRolePrefix+"master"] = "1" - labels[awstasks.CloudTagInstanceGroupRolePrefix+kops.InstanceGroupRoleControlPlane.ToLowerString()] = "1" - } + if ig.Spec.Role == kops.InstanceGroupRoleAPIServer { + labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleAPIServer))] = "1" + } - if ig.Spec.Role == kops.InstanceGroupRoleAPIServer { - labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleAPIServer))] = "1" - } + if ig.Spec.Role == kops.InstanceGroupRoleNode { + labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleNode))] = "1" + } - if ig.Spec.Role == kops.InstanceGroupRoleNode { - labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleNode))] = "1" - } + if ig.Spec.Role == kops.InstanceGroupRoleBastion { + labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleBastion))] = "1" + } - if ig.Spec.Role == kops.InstanceGroupRoleBastion { - labels[awstasks.CloudTagInstanceGroupRolePrefix+strings.ToLower(string(kops.InstanceGroupRoleBastion))] = "1" + labels[nodeidentityaws.CloudTagInstanceGroupName] = ig.Name } - - labels[nodeidentityaws.CloudTagInstanceGroupName] = ig.Name - return labels, nil } @@ -259,6 +271,8 @@ func (b *KopsModelContext) CloudTags(name string, shared bool) map[string]string } tags[k] = v } + case kops.CloudProviderHetzner: + tags[hetzner.TagKubernetesClusterName] = b.ClusterName() } return tags } diff --git a/pkg/model/hetznermodel/servers.go b/pkg/model/hetznermodel/servers.go index a939a79f30cd8..356e588fcc55b 100644 --- a/pkg/model/hetznermodel/servers.go +++ b/pkg/model/hetznermodel/servers.go @@ -54,11 +54,10 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error for _, ig := range b.InstanceGroups { igSize := fi.ValueOf(ig.Spec.MinSize) - - labels := make(map[string]string) - labels[hetzner.TagKubernetesClusterName] = b.ClusterName() - labels[hetzner.TagKubernetesInstanceGroup] = ig.Name - labels[hetzner.TagKubernetesInstanceRole] = string(ig.Spec.Role) + labels, err := b.CloudTagsForInstanceGroup(ig) + if err != nil { + return err + } userData, err := b.BootstrapScriptBuilder.ResourceNodeUp(c, ig) if err != nil { diff --git a/pkg/nodeidentity/hetzner/identify.go b/pkg/nodeidentity/hetzner/identify.go index 4180130ae5b49..70e53cb7cd860 100644 --- a/pkg/nodeidentity/hetzner/identify.go +++ b/pkg/nodeidentity/hetzner/identify.go @@ -97,7 +97,8 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* labels := map[string]string{} for key, value := range server.Labels { - if key == hetzner.TagKubernetesInstanceRole { + switch { + case key == hetzner.TagKubernetesInstanceRole: switch kops.InstanceGroupRole(value) { case kops.InstanceGroupRoleControlPlane: labels[nodelabels.RoleLabelControlPlane20] = "" @@ -108,6 +109,8 @@ func (i *nodeIdentifier) IdentifyNode(ctx context.Context, node *corev1.Node) (* default: klog.Warningf("Unknown node role %q for server %s(%d)", value, server.Name, server.ID) } + case strings.HasPrefix(key, hetzner.TagKubernetesNodeLabelPrefix): + labels[strings.TrimPrefix(key, hetzner.TagKubernetesNodeLabelPrefix)] = value } } diff --git a/upup/pkg/fi/cloudup/hetzner/cloud.go b/upup/pkg/fi/cloudup/hetzner/cloud.go index ec6e5d67a9492..863c51f1af857 100644 --- a/upup/pkg/fi/cloudup/hetzner/cloud.go +++ b/upup/pkg/fi/cloudup/hetzner/cloud.go @@ -41,6 +41,7 @@ const ( TagKubernetesInstanceUserData = "kops.k8s.io/instance-userdata" TagKubernetesInstanceNeedsUpdate = "kops.k8s.io/needs-update" TagKubernetesVolumeRole = "kops.k8s.io/volume-role" + TagKubernetesNodeLabelPrefix = "kops.k8s.io/node-label/" ) // HetznerCloud exposes all the interfaces required to operate on Hetzner Cloud resources From e4fdb2079737adc42645809d9faa006453a6edae Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Tue, 6 Aug 2024 22:31:48 -0500 Subject: [PATCH 2/3] ./hack/update-expected.sh --- .../update_cluster/minimal_hetzner/kubernetes.tf | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf b/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf index c4045deee5254..987de4e89eef5 100644 --- a/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf @@ -235,9 +235,12 @@ resource "hcloud_server" "master-fsn1" { count = 1 image = "ubuntu-20.04" labels = { - "kops.k8s.io/cluster" = "minimal.example.com" - "kops.k8s.io/instance-group" = "master-fsn1" - "kops.k8s.io/instance-role" = "ControlPlane" + "kops.k8s.io/cluster" = "minimal.example.com" + "kops.k8s.io/instance-group" = "master-fsn1" + "kops.k8s.io/instance-role" = "ControlPlane" + "kops.k8s.io/node-label/kops.k8s.io/kops-controller-pki" = "" + "kops.k8s.io/node-label/node-role.kubernetes.io/control-plane" = "" + "kops.k8s.io/node-label/node.kubernetes.io/exclude-from-external-load-balancers" = "" } location = "fsn1" name = "master-fsn1-${count.index}" @@ -257,9 +260,10 @@ resource "hcloud_server" "nodes-fsn1" { count = 1 image = "ubuntu-20.04" labels = { - "kops.k8s.io/cluster" = "minimal.example.com" - "kops.k8s.io/instance-group" = "nodes-fsn1" - "kops.k8s.io/instance-role" = "Node" + "kops.k8s.io/cluster" = "minimal.example.com" + "kops.k8s.io/instance-group" = "nodes-fsn1" + "kops.k8s.io/instance-role" = "Node" + "kops.k8s.io/node-label/node-role.kubernetes.io/node" = "" } location = "fsn1" name = "nodes-fsn1-${count.index}" From d9ebf18dbe69552ea35f20b6eacb9aa4ca7de8ba Mon Sep 17 00:00:00 2001 From: Peter Rifel Date: Wed, 7 Aug 2024 07:40:53 -0500 Subject: [PATCH 3/3] Ensure only one slash in hetzner server labels --- .../update_cluster/minimal_hetzner/kubernetes.tf | 8 ++++---- upup/pkg/fi/cloudup/hetzner/cloud.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf b/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf index 987de4e89eef5..1f30b0db42d4a 100644 --- a/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_hetzner/kubernetes.tf @@ -238,9 +238,9 @@ resource "hcloud_server" "master-fsn1" { "kops.k8s.io/cluster" = "minimal.example.com" "kops.k8s.io/instance-group" = "master-fsn1" "kops.k8s.io/instance-role" = "ControlPlane" - "kops.k8s.io/node-label/kops.k8s.io/kops-controller-pki" = "" - "kops.k8s.io/node-label/node-role.kubernetes.io/control-plane" = "" - "kops.k8s.io/node-label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "node-label.kops.k8s.io.kops.k8s.io/kops-controller-pki" = "" + "node-label.kops.k8s.io.node-role.kubernetes.io/control-plane" = "" + "node-label.kops.k8s.io.node.kubernetes.io/exclude-from-external-load-balancers" = "" } location = "fsn1" name = "master-fsn1-${count.index}" @@ -263,7 +263,7 @@ resource "hcloud_server" "nodes-fsn1" { "kops.k8s.io/cluster" = "minimal.example.com" "kops.k8s.io/instance-group" = "nodes-fsn1" "kops.k8s.io/instance-role" = "Node" - "kops.k8s.io/node-label/node-role.kubernetes.io/node" = "" + "node-label.kops.k8s.io.node-role.kubernetes.io/node" = "" } location = "fsn1" name = "nodes-fsn1-${count.index}" diff --git a/upup/pkg/fi/cloudup/hetzner/cloud.go b/upup/pkg/fi/cloudup/hetzner/cloud.go index 863c51f1af857..e23701376b760 100644 --- a/upup/pkg/fi/cloudup/hetzner/cloud.go +++ b/upup/pkg/fi/cloudup/hetzner/cloud.go @@ -41,7 +41,7 @@ const ( TagKubernetesInstanceUserData = "kops.k8s.io/instance-userdata" TagKubernetesInstanceNeedsUpdate = "kops.k8s.io/needs-update" TagKubernetesVolumeRole = "kops.k8s.io/volume-role" - TagKubernetesNodeLabelPrefix = "kops.k8s.io/node-label/" + TagKubernetesNodeLabelPrefix = "node-label.kops.k8s.io." ) // HetznerCloud exposes all the interfaces required to operate on Hetzner Cloud resources