diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 054e5a60e50fa..cbd2ca4498345 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -314,7 +314,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.EtcdStorageType, "etcd-storage-type", options.EtcdStorageType, "The default storage type for etcd members") cmd.RegisterFlagCompletionFunc("etcd-storage-type", completeStorageType) - cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni.") + cmd.Flags().StringVar(&options.Networking, "networking", options.Networking, "Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, kindnet, cni.") cmd.RegisterFlagCompletionFunc("networking", completeNetworking(options)) cmd.Flags().StringVar(&options.DNSZone, "dns-zone", options.DNSZone, "DNS hosted zone (defaults to longest matching zone)") @@ -983,6 +983,7 @@ func completeNetworking(options *CreateClusterOptions) func(cmd *cobra.Command, "cilium", "cilium-eni", "cilium-etcd", + "kindnet", } if !options.IPv6 { diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index ca90a353f72a6..99fbb62efef02 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -228,6 +228,7 @@ const ( canalAddon = "networking.projectcalico.org.canal-k8s-1.25" ciliumAddon = "networking.cilium.io-k8s-1.16" flannelAddon = "networking.flannel-k8s-1.25" + kindnetAddon = "networking.kindnet-k8s-1.12" certManagerAddon = "certmanager.io-k8s-1.16" clusterAutoscalerAddon = "cluster-autoscaler.addons.k8s.io-k8s-1.15" @@ -642,6 +643,15 @@ func TestPrivateFlannel(t *testing.T) { runTestTerraformAWS(t) } +// TestPrivateKindnet runs the test on a configuration with private topology, flannel networking +func TestPrivateKindnet(t *testing.T) { + newIntegrationTest("privatekindnet.example.com", "privatekindnet"). + withPrivate(). + withDefaultAddons24(). + withAddons(kindnetAddon). + runTestTerraformAWS(t) +} + // TestPrivateCalico runs the test on a configuration with private topology, calico networking func TestPrivateCalico(t *testing.T) { newIntegrationTest("privatecalico.example.com", "privatecalico"). diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 8bde12799ae2f..13bae47971217 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -100,7 +100,7 @@ kops create cluster [CLUSTER] [flags] --kubernetes-version string Version of Kubernetes to run (defaults to version in channel) --network-cidr strings Network CIDR(s) to use --network-id string Shared Network or VPC to use - --networking string Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, cni. (default "cilium") + --networking string Networking mode. kubenet, external, flannel-vxlan (or flannel), flannel-udp, calico, canal, kube-router, amazonvpc, cilium, cilium-etcd, kindnet, cni. (default "cilium") --node-count int32 Total number of worker nodes. Defaults to one node per zone --node-image string Machine image for worker nodes. Takes precedence over --image --node-security-groups strings Additional pre-created security groups to add to worker nodes. diff --git a/docs/networking/kindnet.md b/docs/networking/kindnet.md new file mode 100644 index 0000000000000..b85e2f4bd2c11 --- /dev/null +++ b/docs/networking/kindnet.md @@ -0,0 +1,26 @@ +# Kindnet + +* [kindnet](http://kindnet.es) + +Kindnet focuses on providing essential networking functionality without unnecessary complexity. + +## Installing + +To install [kindnet](https://github.com/aojea/kindnet) - use `--networking kindnet`. + +```sh +export ZONES=mylistofzone +kops create cluster \ + --zones $ZONES \ + --networking kindnet \ + --yes \ + --name myclustername.mydns.io +``` + +## Getting help + +For problems with kindnet please post an issue to Github: + +- [Kindnet Issues](https://github.com/aojea/kindnet/issues) + +You can learn more about the different configurations options in https://kindnet.es/ diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 782ee35692309..f7c35e9cc289a 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -5773,6 +5773,36 @@ spec: description: GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type: object + kindnet: + description: KindnetNetworkingSpec configures Kindnet settings. + properties: + adminNetworkPolicies: + type: boolean + baselineAdminNetworkPolicies: + type: boolean + dnsCaching: + type: boolean + fastPathThreshold: + format: int32 + type: integer + masquerade: + description: KindnetMasqueradeSpec configures Kindnet masquerading + settings. + properties: + enabled: + type: boolean + nonMasqueradeCIDRs: + items: + type: string + type: array + type: object + nat64: + type: boolean + networkPolicies: + type: boolean + version: + type: string + type: object kopeio: description: KopeioNetworkingSpec declares that we want Kopeio networking diff --git a/nodeup/pkg/model/networking/kindnet.go b/nodeup/pkg/model/networking/kindnet.go new file mode 100644 index 0000000000000..c13f2cd9eafc4 --- /dev/null +++ b/nodeup/pkg/model/networking/kindnet.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package networking + +import ( + "k8s.io/kops/nodeup/pkg/model" + "k8s.io/kops/upup/pkg/fi" +) + +// KindnetBuilder configures the etcd TLS support for Calico +type KindnetBuilder struct { + *model.NodeupModelContext +} + +var _ fi.NodeupModelBuilder = &KindnetBuilder{} + +// Build is responsible for performing setup for Kindnet. +func (b *KindnetBuilder) Build(c *fi.NodeupModelBuilderContext) error { + if b.NodeupConfig.Networking.Kindnet == nil { + return nil + } + + return nil +} diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 3edb9b2c20eac..1882d226d70b2 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -78,6 +78,7 @@ type NetworkingSpec struct { Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` GCP *GCPNetworkingSpec `json:"gcp,omitempty"` + Kindnet *KindnetNetworkingSpec `json:"kindnet,omitempty"` } // UsesKubenet returns true if our networking is derived from kubenet @@ -553,3 +554,27 @@ type LyftVPCNetworkingSpec struct { // GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type GCPNetworkingSpec struct{} + +// KindnetNetworkingSpec configures Kindnet settings. +type KindnetNetworkingSpec struct { + // Version is the version of the kindnet agent. + // Default: stable + Version string `json:"version,omitempty"` + // Enable network policies + NetworkPolicies *bool `json:"networkPolicies,omitempty"` + AdminNetworkPolicies *bool `json:"adminNetworkPolicies,omitempty"` + BaselineAdminNetworkPolicies *bool `json:"baselineAdminNetworkPolicies,omitempty"` + // enable dns caching + DNSCaching *bool `json:"dnsCaching,omitempty"` + // enable nat64 on ipv6 clusters + NAT64 *bool `json:"nat64,omitempty"` + // number of packets in a connection to offload it to the fast path + FastPathThreshold *int32 `json:"fastPathThreshold,omitempty"` + // node agent masquerading rules + Masquerade *KindnetMasqueradeSpec `json:"masquerade,omitempty"` +} + +type KindnetMasqueradeSpec struct { + Enabled *bool `json:"enabled,omitempty"` + NonMasqueradeCIDRs []string `json:"nonMasqueradeCIDRs,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index 09af2790af39a..8b131363eb6be 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -49,12 +49,13 @@ type NetworkingSpec struct { Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` LyftVPC *LyftVPCNetworkingSpec `json:"lyftvpc,omitempty"` GCP *GCPNetworkingSpec `json:"gce,omitempty"` + Kindnet *KindnetNetworkingSpec `json:"kindnet,omitempty"` } func (s *NetworkingSpec) IsEmpty() bool { return s.Classic == nil && s.Kubenet == nil && s.External == nil && s.CNI == nil && s.Kopeio == nil && s.Weave == nil && s.Flannel == nil && s.Calico == nil && s.Canal == nil && s.KubeRouter == nil && - s.Romana == nil && s.AmazonVPC == nil && s.Cilium == nil && s.LyftVPC == nil && s.GCP == nil + s.Romana == nil && s.AmazonVPC == nil && s.Cilium == nil && s.LyftVPC == nil && s.GCP == nil && s.Kindnet == nil } // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes. @@ -674,3 +675,21 @@ type LyftVPCNetworkingSpec struct { // GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type GCPNetworkingSpec struct{} + +// KindnetNetworkingSpec configures Kindnet settings. +type KindnetNetworkingSpec struct { + Version string `json:"version,omitempty"` + NetworkPolicies *bool `json:"networkPolicies,omitempty"` + AdminNetworkPolicies *bool `json:"adminNetworkPolicies,omitempty"` + BaselineAdminNetworkPolicies *bool `json:"baselineAdminNetworkPolicies,omitempty"` + DNSCaching *bool `json:"dnsCaching,omitempty"` + NAT64 *bool `json:"nat64,omitempty"` + FastPathThreshold *int32 `json:"fastPathThreshold,omitempty"` + Masquerade *KindnetMasqueradeSpec `json:"masquerade,omitempty"` +} + +// KindnetMasqueradeSpec configures Kindnet masquerading settings. +type KindnetMasqueradeSpec struct { + Enabled *bool `json:"enabled,omitempty"` + NonMasqueradeCIDRs []string `json:"nonMasqueradeCIDRs,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 898c243e04dc0..c727a01955ff1 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -624,6 +624,26 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KindnetMasqueradeSpec)(nil), (*kops.KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(a.(*KindnetMasqueradeSpec), b.(*kops.KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetMasqueradeSpec)(nil), (*KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(a.(*kops.KindnetMasqueradeSpec), b.(*KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*KindnetNetworkingSpec)(nil), (*kops.KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(a.(*KindnetNetworkingSpec), b.(*kops.KindnetNetworkingSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetNetworkingSpec)(nil), (*KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(a.(*kops.KindnetNetworkingSpec), b.(*KindnetNetworkingSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KopeioAuthenticationSpec)(nil), (*kops.KopeioAuthenticationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(a.(*KopeioAuthenticationSpec), b.(*kops.KopeioAuthenticationSpec), scope) }); err != nil { @@ -4831,6 +4851,78 @@ func Convert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in *kops.KeysetSpec, out *Ke return autoConvert_kops_KeysetSpec_To_v1alpha2_KeysetSpec(in, out, s) } +func autoConvert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(kops.KindnetMasqueradeSpec) + if err := Convert_v1alpha2_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + return nil +} + +// Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in, out, s) +} + +func autoConvert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + if err := Convert_kops_KindnetMasqueradeSpec_To_v1alpha2_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + return nil +} + +// Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(in, out, s) +} + func autoConvert_v1alpha2_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(in *KopeioAuthenticationSpec, out *kops.KopeioAuthenticationSpec, s conversion.Scope) error { return nil } @@ -6224,6 +6316,15 @@ func autoConvert_v1alpha2_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSp } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(kops.KindnetNetworkingSpec) + if err := Convert_v1alpha2_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } @@ -6405,6 +6506,15 @@ func autoConvert_kops_NetworkingSpec_To_v1alpha2_NetworkingSpec(in *kops.Network } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + if err := Convert_kops_KindnetNetworkingSpec_To_v1alpha2_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index 99f85e64b9a70..0933bf01c6d87 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -3011,6 +3011,83 @@ func (in *KeysetSpec) DeepCopy() *KeysetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetMasqueradeSpec) DeepCopyInto(out *KindnetMasqueradeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NonMasqueradeCIDRs != nil { + in, out := &in.NonMasqueradeCIDRs, &out.NonMasqueradeCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetMasqueradeSpec. +func (in *KindnetMasqueradeSpec) DeepCopy() *KindnetMasqueradeSpec { + if in == nil { + return nil + } + out := new(KindnetMasqueradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetNetworkingSpec) DeepCopyInto(out *KindnetNetworkingSpec) { + *out = *in + if in.NetworkPolicies != nil { + in, out := &in.NetworkPolicies, &out.NetworkPolicies + *out = new(bool) + **out = **in + } + if in.AdminNetworkPolicies != nil { + in, out := &in.AdminNetworkPolicies, &out.AdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.BaselineAdminNetworkPolicies != nil { + in, out := &in.BaselineAdminNetworkPolicies, &out.BaselineAdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.DNSCaching != nil { + in, out := &in.DNSCaching, &out.DNSCaching + *out = new(bool) + **out = **in + } + if in.NAT64 != nil { + in, out := &in.NAT64, &out.NAT64 + *out = new(bool) + **out = **in + } + if in.FastPathThreshold != nil { + in, out := &in.FastPathThreshold, &out.FastPathThreshold + *out = new(int32) + **out = **in + } + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetNetworkingSpec. +func (in *KindnetNetworkingSpec) DeepCopy() *KindnetNetworkingSpec { + if in == nil { + return nil + } + out := new(KindnetNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopeioAuthenticationSpec) DeepCopyInto(out *KopeioAuthenticationSpec) { *out = *in @@ -4783,6 +4860,11 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = new(GCPNetworkingSpec) **out = **in } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/kops/v1alpha3/networking.go b/pkg/apis/kops/v1alpha3/networking.go index 2ccb675bf445f..7d871dde748e9 100644 --- a/pkg/apis/kops/v1alpha3/networking.go +++ b/pkg/apis/kops/v1alpha3/networking.go @@ -82,6 +82,7 @@ type NetworkingSpec struct { Cilium *CiliumNetworkingSpec `json:"cilium,omitempty"` LyftVPC *kops.LyftVPCNetworkingSpec `json:"-"` GCP *GCPNetworkingSpec `json:"gcp,omitempty"` + Kindnet *KindnetNetworkingSpec `json:"kindnet,omitempty"` } // KubenetNetworkingSpec is the specification for kubenet networking, largely integrated but intended to replace classic @@ -510,3 +511,20 @@ type HubbleSpec struct { // GCPNetworkingSpec is the specification of GCP's native networking mode, using IP aliases. type GCPNetworkingSpec struct{} + +// KindnetNetworkingSpec configures Kindnet settings. +type KindnetNetworkingSpec struct { + Version string `json:"version,omitempty"` + NetworkPolicies *bool `json:"networkPolicies,omitempty"` + AdminNetworkPolicies *bool `json:"adminNetworkPolicies,omitempty"` + BaselineAdminNetworkPolicies *bool `json:"baselineAdminNetworkPolicies,omitempty"` + DNSCaching *bool `json:"dnsCaching,omitempty"` + NAT64 *bool `json:"nat64,omitempty"` + FastPathThreshold *int32 `json:"fastPathThreshold,omitempty"` + Masquerade *KindnetMasqueradeSpec `json:"masquerade,omitempty"` +} + +type KindnetMasqueradeSpec struct { + Enabled *bool `json:"enabled,omitempty"` + NonMasqueradeCIDRs []string `json:"nonMasqueradeCIDRs,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go index b979e1e79ae7b..89a220bc33369 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go @@ -744,6 +744,26 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*KindnetMasqueradeSpec)(nil), (*kops.KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(a.(*KindnetMasqueradeSpec), b.(*kops.KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetMasqueradeSpec)(nil), (*KindnetMasqueradeSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(a.(*kops.KindnetMasqueradeSpec), b.(*KindnetMasqueradeSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*KindnetNetworkingSpec)(nil), (*kops.KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(a.(*KindnetNetworkingSpec), b.(*kops.KindnetNetworkingSpec), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.KindnetNetworkingSpec)(nil), (*KindnetNetworkingSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(a.(*kops.KindnetNetworkingSpec), b.(*KindnetNetworkingSpec), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*KopeioAuthenticationSpec)(nil), (*kops.KopeioAuthenticationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha3_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(a.(*KopeioAuthenticationSpec), b.(*kops.KopeioAuthenticationSpec), scope) }); err != nil { @@ -5227,6 +5247,78 @@ func Convert_kops_KeysetSpec_To_v1alpha3_KeysetSpec(in *kops.KeysetSpec, out *Ke return autoConvert_kops_KeysetSpec_To_v1alpha3_KeysetSpec(in, out, s) } +func autoConvert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in *KindnetMasqueradeSpec, out *kops.KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + out.Enabled = in.Enabled + out.NonMasqueradeCIDRs = in.NonMasqueradeCIDRs + return nil +} + +// Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec is an autogenerated conversion function. +func Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(in *kops.KindnetMasqueradeSpec, out *KindnetMasqueradeSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(in, out, s) +} + +func autoConvert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(kops.KindnetMasqueradeSpec) + if err := Convert_v1alpha3_KindnetMasqueradeSpec_To_kops_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + return nil +} + +// Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in *KindnetNetworkingSpec, out *kops.KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(in, out, s) +} + +func autoConvert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + out.Version = in.Version + out.NetworkPolicies = in.NetworkPolicies + out.AdminNetworkPolicies = in.AdminNetworkPolicies + out.BaselineAdminNetworkPolicies = in.BaselineAdminNetworkPolicies + out.DNSCaching = in.DNSCaching + out.NAT64 = in.NAT64 + out.FastPathThreshold = in.FastPathThreshold + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + if err := Convert_kops_KindnetMasqueradeSpec_To_v1alpha3_KindnetMasqueradeSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Masquerade = nil + } + return nil +} + +// Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec is an autogenerated conversion function. +func Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(in *kops.KindnetNetworkingSpec, out *KindnetNetworkingSpec, s conversion.Scope) error { + return autoConvert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(in, out, s) +} + func autoConvert_v1alpha3_KopeioAuthenticationSpec_To_kops_KopeioAuthenticationSpec(in *KopeioAuthenticationSpec, out *kops.KopeioAuthenticationSpec, s conversion.Scope) error { return nil } @@ -6574,6 +6666,15 @@ func autoConvert_v1alpha3_NetworkingSpec_To_kops_NetworkingSpec(in *NetworkingSp } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(kops.KindnetNetworkingSpec) + if err := Convert_v1alpha3_KindnetNetworkingSpec_To_kops_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } @@ -6731,6 +6832,15 @@ func autoConvert_kops_NetworkingSpec_To_v1alpha3_NetworkingSpec(in *kops.Network } else { out.GCP = nil } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + if err := Convert_kops_KindnetNetworkingSpec_To_v1alpha3_KindnetNetworkingSpec(*in, *out, s); err != nil { + return err + } + } else { + out.Kindnet = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go index 0a84da4d2adec..3a7efd7fa1793 100644 --- a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go @@ -2985,6 +2985,83 @@ func (in *KeysetSpec) DeepCopy() *KeysetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetMasqueradeSpec) DeepCopyInto(out *KindnetMasqueradeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NonMasqueradeCIDRs != nil { + in, out := &in.NonMasqueradeCIDRs, &out.NonMasqueradeCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetMasqueradeSpec. +func (in *KindnetMasqueradeSpec) DeepCopy() *KindnetMasqueradeSpec { + if in == nil { + return nil + } + out := new(KindnetMasqueradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetNetworkingSpec) DeepCopyInto(out *KindnetNetworkingSpec) { + *out = *in + if in.NetworkPolicies != nil { + in, out := &in.NetworkPolicies, &out.NetworkPolicies + *out = new(bool) + **out = **in + } + if in.AdminNetworkPolicies != nil { + in, out := &in.AdminNetworkPolicies, &out.AdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.BaselineAdminNetworkPolicies != nil { + in, out := &in.BaselineAdminNetworkPolicies, &out.BaselineAdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.DNSCaching != nil { + in, out := &in.DNSCaching, &out.DNSCaching + *out = new(bool) + **out = **in + } + if in.NAT64 != nil { + in, out := &in.NAT64, &out.NAT64 + *out = new(bool) + **out = **in + } + if in.FastPathThreshold != nil { + in, out := &in.FastPathThreshold, &out.FastPathThreshold + *out = new(int32) + **out = **in + } + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetNetworkingSpec. +func (in *KindnetNetworkingSpec) DeepCopy() *KindnetNetworkingSpec { + if in == nil { + return nil + } + out := new(KindnetNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopeioAuthenticationSpec) DeepCopyInto(out *KopeioAuthenticationSpec) { *out = *in @@ -4734,6 +4811,11 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = new(GCPNetworkingSpec) **out = **in } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index a3b303b7edf85..76685ed7b6883 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "net" + "net/netip" "net/url" "path/filepath" "regexp" @@ -1226,6 +1227,14 @@ func validateNetworking(cluster *kops.Cluster, v *kops.NetworkingSpec, fldPath * allErrs = append(allErrs, validateNetworkingGCP(cluster, v.GCP, fldPath.Child("gcp"))...) } + if v.Kindnet != nil { + if optionTaken { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("kindnet"), "only one networking option permitted")) + } + + allErrs = append(allErrs, validateNetworkingKindnet(cluster, v.Kindnet, fldPath.Child("kindnet"))...) + } + return allErrs } @@ -1403,6 +1412,23 @@ func validateNetworkingGCP(cluster *kops.Cluster, v *kops.GCPNetworkingSpec, fld return allErrs } +func validateNetworkingKindnet(cluster *kops.Cluster, v *kops.KindnetNetworkingSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if v.Masquerade != nil && v.Masquerade.Enabled != nil && *v.Masquerade.Enabled { + for _, cidr := range v.Masquerade.NonMasqueradeCIDRs { + if cidr == "" { + continue + } + _, err := netip.ParsePrefix(cidr) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, cidr, err.Error())) + } + } + } + return allErrs +} + func validateAdditionalPolicy(role string, policy string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index da2136c5c9db4..1c6b072f2b0ba 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" + "k8s.io/utils/ptr" ) func Test_Validate_DNS(t *testing.T) { @@ -458,6 +459,72 @@ func Test_Validate_Networking_Flannel(t *testing.T) { } } +func Test_Validate_Networking_Kindnet(t *testing.T) { + grid := []struct { + Input kops.KindnetNetworkingSpec + ExpectedErrors []string + }{ + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(true), + }, + }, + }, + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(true), + NonMasqueradeCIDRs: []string{"10.0.0.0/24", "2001:db8::/64"}, + }, + }, + }, + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(true), + NonMasqueradeCIDRs: []string{"a.b.c.d/24", "2001:db8::/64"}, + }, + }, + ExpectedErrors: []string{"Invalid value::networking.kindnet"}, + }, + { + Input: kops.KindnetNetworkingSpec{ + Masquerade: &kops.KindnetMasqueradeSpec{ + Enabled: ptr.To(false), + NonMasqueradeCIDRs: []string{"a.b.c.d/24", "2001:db8::/64"}, + }, + }, + ExpectedErrors: []string{}, + }, + } + + for _, g := range grid { + cluster := &kops.Cluster{ + Spec: kops.ClusterSpec{ + KubernetesVersion: "1.27.0", + Networking: kops.NetworkingSpec{ + NetworkCIDR: "10.0.0.0/8", + NonMasqueradeCIDR: "100.64.0.0/10", + PodCIDR: "100.96.0.0/11", + ServiceClusterIPRange: "100.64.0.0/13", + Subnets: []kops.ClusterSubnetSpec{ + { + Name: "sg-test", + CIDR: "10.11.0.0/16", + Type: "Public", + }, + }, + Kindnet: &g.Input, + }, + }, + } + + errs := validateNetworking(cluster, &cluster.Spec.Networking, field.NewPath("networking"), true, &cloudProviderConstraints{}) + testErrors(t, g.Input, errs, g.ExpectedErrors) + } +} + func Test_Validate_Networking_OverlappingCIDR(t *testing.T) { grid := []struct { Name string diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 87ef7670a9792..5ba62bd94eebd 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -3072,6 +3072,83 @@ func (in *KeysetSpec) DeepCopy() *KeysetSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetMasqueradeSpec) DeepCopyInto(out *KindnetMasqueradeSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.NonMasqueradeCIDRs != nil { + in, out := &in.NonMasqueradeCIDRs, &out.NonMasqueradeCIDRs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetMasqueradeSpec. +func (in *KindnetMasqueradeSpec) DeepCopy() *KindnetMasqueradeSpec { + if in == nil { + return nil + } + out := new(KindnetMasqueradeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KindnetNetworkingSpec) DeepCopyInto(out *KindnetNetworkingSpec) { + *out = *in + if in.NetworkPolicies != nil { + in, out := &in.NetworkPolicies, &out.NetworkPolicies + *out = new(bool) + **out = **in + } + if in.AdminNetworkPolicies != nil { + in, out := &in.AdminNetworkPolicies, &out.AdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.BaselineAdminNetworkPolicies != nil { + in, out := &in.BaselineAdminNetworkPolicies, &out.BaselineAdminNetworkPolicies + *out = new(bool) + **out = **in + } + if in.DNSCaching != nil { + in, out := &in.DNSCaching, &out.DNSCaching + *out = new(bool) + **out = **in + } + if in.NAT64 != nil { + in, out := &in.NAT64, &out.NAT64 + *out = new(bool) + **out = **in + } + if in.FastPathThreshold != nil { + in, out := &in.FastPathThreshold, &out.FastPathThreshold + *out = new(int32) + **out = **in + } + if in.Masquerade != nil { + in, out := &in.Masquerade, &out.Masquerade + *out = new(KindnetMasqueradeSpec) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KindnetNetworkingSpec. +func (in *KindnetNetworkingSpec) DeepCopy() *KindnetNetworkingSpec { + if in == nil { + return nil + } + out := new(KindnetNetworkingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KopeioAuthenticationSpec) DeepCopyInto(out *KopeioAuthenticationSpec) { *out = *in @@ -4876,6 +4953,11 @@ func (in *NetworkingSpec) DeepCopyInto(out *NetworkingSpec) { *out = new(GCPNetworkingSpec) **out = **in } + if in.Kindnet != nil { + in, out := &in.Kindnet, &out.Kindnet + *out = new(KindnetNetworkingSpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/nodeup/config.go b/pkg/apis/nodeup/config.go index 1386e8028410d..7c6ccb8b7a10c 100644 --- a/pkg/apis/nodeup/config.go +++ b/pkg/apis/nodeup/config.go @@ -328,6 +328,10 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi config.Networking.KubeRouter = &kops.KuberouterNetworkingSpec{} } + if cluster.Spec.Networking.Kindnet != nil { + config.Networking.Kindnet = &kops.KindnetNetworkingSpec{} + } + if instanceGroup.Spec.Kubelet != nil { config.KubeletConfig = *instanceGroup.Spec.Kubelet } diff --git a/pkg/model/components/kindnet.go b/pkg/model/components/kindnet.go new file mode 100644 index 0000000000000..da72dcc94ec86 --- /dev/null +++ b/pkg/model/components/kindnet.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/loader" +) + +// KindnetOptionsBuilder adds options for kindnet to the model +type KindnetOptionsBuilder struct { + Context *OptionsContext +} + +var _ loader.ClusterOptionsBuilder = &KindnetOptionsBuilder{} + +func (b *KindnetOptionsBuilder) BuildOptions(o *kops.Cluster) error { + clusterSpec := &o.Spec + c := clusterSpec.Networking.Kindnet + if c == nil { + return nil + } + + if c.Version == "" { + c.Version = "stable" + } + + if c.Masquerade == nil { + c.Masquerade = &kops.KindnetMasqueradeSpec{ + Enabled: fi.PtrTo(true), + } + if clusterSpec.Networking.NetworkCIDR != "" { + c.Masquerade.NonMasqueradeCIDRs = append(c.Masquerade.NonMasqueradeCIDRs, clusterSpec.Networking.NetworkCIDR) + } + if clusterSpec.Networking.PodCIDR != "" { + c.Masquerade.NonMasqueradeCIDRs = append(c.Masquerade.NonMasqueradeCIDRs, clusterSpec.Networking.PodCIDR) + } + if clusterSpec.Networking.ServiceClusterIPRange != "" { + c.Masquerade.NonMasqueradeCIDRs = append(c.Masquerade.NonMasqueradeCIDRs, clusterSpec.Networking.ServiceClusterIPRange) + } + } + + return nil +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern new file mode 100644 index 0000000000000..c8db9dbe9c41b --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern @@ -0,0 +1 @@ +{"source":["aws.autoscaling"],"detail-type":["EC2 Instance-terminate Lifecycle Action"]} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern new file mode 100644 index 0000000000000..fb4ea7defdc52 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.health"],"detail-type": ["AWS Health Event"],"detail": {"service": ["EC2"],"eventTypeCategory": ["scheduledChange"]}} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern new file mode 100644 index 0000000000000..8c2916419dc7d --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.ec2"],"detail-type": ["EC2 Instance State-change Notification"]} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern new file mode 100644 index 0000000000000..2d0e83b4164b2 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern @@ -0,0 +1 @@ +{"source": ["aws.ec2"],"detail-type": ["EC2 Spot Instance Interruption Warning"]} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_bastions.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_masters.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_nodes.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..54912e12a5880 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy @@ -0,0 +1,10 @@ +{ + "Statement": [ + { + "Action": "ec2:DescribeRegions", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..1c3c73ee5e95a --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy @@ -0,0 +1,276 @@ +{ + "Statement": [ + { + "Action": "ec2:AttachVolume", + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com", + "aws:ResourceTag/k8s.io/role/master": "1" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "s3:Get*" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-read-bucket/clusters.example.com/privatekindnet.example.com/*" + }, + { + "Action": [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/privatekindnet.example.com/backups/etcd/main/*" + }, + { + "Action": [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": "arn:aws-test:s3:::placeholder-write-bucket/clusters.example.com/privatekindnet.example.com/backups/etcd/events/*" + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-write-bucket" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:GetHostedZone", + "route53:ListResourceRecordSets" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones", + "route53:ListTagsForResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privatekindnet.example.com", + "ec2:CreateAction": [ + "CreateVolume", + "CreateSnapshot" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:snapshot/*", + "arn:aws-test:ec2:*:*:volume/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:snapshot/*", + "arn:aws-test:ec2:*:*:volume/*" + ] + }, + { + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privatekindnet.example.com", + "ec2:CreateAction": [ + "CreateSecurityGroup" + ] + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Condition": { + "Null": { + "aws:RequestTag/KubernetesCluster": "true" + }, + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "arn:aws-test:ec2:*:*:security-group/*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", + "autoscaling:DescribeTags", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeVolumesModifications", + "ec2:DescribeVpcs", + "ec2:GetInstanceTypesFromInstanceRequirements", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "iam:CreateServiceLinkedRole", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:GenerateRandom", + "kms:ReEncrypt*", + "sqs:DeleteMessage", + "sqs:ReceiveMessage" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "autoscaling:CompleteLifecycleAction", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:RevokeSecurityGroupIngress", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateSnapshot", + "ec2:CreateVolume", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/KubernetesCluster": "privatekindnet.example.com" + } + }, + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "ec2:CreateSecurityGroup", + "Effect": "Allow", + "Resource": "arn:aws-test:ec2:*:*:vpc/*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy new file mode 100644 index 0000000000000..153ab3c7f64f4 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy @@ -0,0 +1,30 @@ +{ + "Statement": [ + { + "Action": [ + "s3:GetBucketLocation", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:ListBucketVersions" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws-test:s3:::placeholder-read-bucket" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + "kms:GenerateRandom" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key new file mode 100644 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data new file mode 100644 index 0000000000000..128fb88c3f69c --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data @@ -0,0 +1,138 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + mapfile -t urls < <(split-commas "$3") + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo "$1" | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ClusterName: privatekindnet.example.com +ConfigBase: memfs://clusters.example.com/privatekindnet.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: ControlPlane +NodeupConfigHash: /K7KjHW6CoQVq3llLrUnTXCVITmCP9vKlAUUrtqsmrM= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data new file mode 100644 index 0000000000000..b8aa7c8344dac --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_launch_template_nodes.privatekindnet.example.com_user_data @@ -0,0 +1,161 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=585fbda0f0a43184656b4bfc0cc5f0c0b85612faf43b8816acca1f99d422c924 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=7603675379699105a9b9915ff97718ea99b1bbb01a4c184e2f827c8a96e8e865 + +export AWS_REGION=us-test-1 + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + mapfile -t urls < <(split-commas "$3") + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo "$1" | tr "," "\n" +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: aws +ClusterName: privatekindnet.example.com +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.privatekindnet.example.com:3988/ +InstanceGroupName: nodes +InstanceGroupRole: Node +NodeupConfigHash: Vub5x6TX4/z4qoFr0s4UIKi8e5LaMUbl07+3LilHovs= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content new file mode 100644 index 0000000000000..d1095f4a75126 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_cluster-completed.spec_content @@ -0,0 +1,232 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-12T04:13:14Z" + name: privatekindnet.example.com +spec: + api: + loadBalancer: + class: Classic + type: Public + authorization: + alwaysAllow: {} + channel: stable + cloudConfig: + awsEBSCSIDriver: + version: v1.33.0 + manageStorageClasses: true + cloudControllerManager: + allocateNodeCIDRs: true + clusterCIDR: 100.96.0.0/11 + clusterName: privatekindnet.example.com + configureCloudRoutes: false + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.25.15 + leaderElection: + leaderElect: true + cloudProvider: aws + clusterDNSDomain: cluster.local + configBase: memfs://clusters.example.com/privatekindnet.example.com + containerd: + logLevel: info + runc: + version: 1.1.5 + version: 1.6.20 + dnsZone: Z1AFAKE1ZON3YO + etcdClusters: + - backups: + backupStore: memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/main + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + manager: + backupRetentionDays: 90 + name: main + version: 3.5.13 + - backups: + backupStore: memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/events + etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + manager: + backupRetentionDays: 90 + name: events + version: 3.5.13 + externalDns: + provider: dns-controller + iam: + legacy: false + keyStore: memfs://clusters.example.com/privatekindnet.example.com/pki + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privatekindnet.example.com + serviceAccountJWKSURI: https://api.internal.privatekindnet.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: privatekindnet.example.com + configureCloudRoutes: false + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.25.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeDNS: + cacheMaxConcurrent: 150 + cacheMaxSize: 1000 + cpuRequest: 100m + domain: cluster.local + memoryLimit: 170Mi + memoryRequest: 70Mi + nodeLocalDNS: + cpuRequest: 25m + enabled: false + image: registry.k8s.io/dns/k8s-dns-node-cache:1.23.0 + memoryRequest: 5Mi + provider: CoreDNS + serverIP: 100.64.0.10 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.25.0 + logLevel: 2 + kubeScheduler: + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.25.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: 1.25.0 + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + masterPublicName: api.privatekindnet.example.com + networkCIDR: 172.20.0.0/16 + networking: + kindnet: + masquerade: + enabled: true + nonMasqueradeCIDRs: + - 172.20.0.0/16 + - 100.96.0.0/11 + - 100.64.0.0/13 + version: stable + nodeTerminationHandler: + cpuRequest: 50m + deleteSQSMsgIfNodeNotFound: false + enableRebalanceDraining: false + enableRebalanceMonitoring: false + enableScheduledEventDraining: true + enableSpotInterruptionDraining: true + enabled: true + excludeFromLoadBalancers: true + managedASGTag: aws-node-termination-handler/managed + memoryRequest: 64Mi + podTerminationGracePeriod: -1 + prometheusEnable: false + taintNode: false + version: v1.22.0 + nonMasqueradeCIDR: 100.64.0.0/10 + podCIDR: 100.96.0.0/11 + secretStore: memfs://clusters.example.com/privatekindnet.example.com/secrets + serviceClusterIPRange: 100.64.0.0/13 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Private + zone: us-test-1a + - cidr: 172.20.4.0/22 + name: utility-us-test-1a + type: Utility + zone: us-test-1a + topology: + dns: + type: Public diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content new file mode 100644 index 0000000000000..5a36c3ebf46dc --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-events_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.5.13" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content new file mode 100644 index 0000000000000..5a36c3ebf46dc --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_etcd-cluster-spec-main_content @@ -0,0 +1,4 @@ +{ + "memberCount": 1, + "etcdVersion": "3.5.13" +} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content new file mode 100644 index 0000000000000..b7340298dcdd5 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_kops-version.txt_content @@ -0,0 +1 @@ +1.21.0-alpha.1 diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content new file mode 100644 index 0000000000000..7641b174f0533 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.privatekindnet.example.com --grpc-port=3997 --peer-urls=https://__name__:2381 + --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s.io/etcd/events + --volume-provider=aws --volume-tag=k8s.io/etcd/events --volume-tag=k8s.io/role/control-plane=1 + --volume-tag=kubernetes.io/cluster/privatekindnet.example.com=owned > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20241012 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + initContainers: + - args: + - --target-dir=/opt/kops-utils/ + - --src=/ko-app/kops-utils-cp + command: + - /ko-app/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: kops-utils-cp + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.4.13 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.4.13-0 + name: init-etcd-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.5.13 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.5.13-0 + name: init-etcd-3-5-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.4.3 + - --src=/opt/etcd-v3.4.13/etcd + - --src=/opt/etcd-v3.4.13/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.5.0 + - --target-dir=/opt/etcd-v3.5.1 + - --target-dir=/opt/etcd-v3.5.3 + - --target-dir=/opt/etcd-v3.5.4 + - --target-dir=/opt/etcd-v3.5.6 + - --target-dir=/opt/etcd-v3.5.7 + - --target-dir=/opt/etcd-v3.5.9 + - --src=/opt/etcd-v3.5.13/etcd + - --src=/opt/etcd-v3.5.13/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-5-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content new file mode 100644 index 0000000000000..b693a4081cfd8 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /etcd-manager + --backup-store=memfs://clusters.example.com/privatekindnet.example.com/backups/etcd/main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.privatekindnet.example.com --grpc-port=3996 --peer-urls=https://__name__:2380 + --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s.io/etcd/main + --volume-provider=aws --volume-tag=k8s.io/etcd/main --volume-tag=k8s.io/role/control-plane=1 + --volume-tag=kubernetes.io/cluster/privatekindnet.example.com=owned > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20241012 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + initContainers: + - args: + - --target-dir=/opt/kops-utils/ + - --src=/ko-app/kops-utils-cp + command: + - /ko-app/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: kops-utils-cp + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.4.13 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.4.13-0 + name: init-etcd-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --target-dir=/opt/etcd-v3.5.13 + - --src=/usr/local/bin/etcd + - --src=/usr/local/bin/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/etcd:3.5.13-0 + name: init-etcd-3-5-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.4.3 + - --src=/opt/etcd-v3.4.13/etcd + - --src=/opt/etcd-v3.4.13/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-4-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + - args: + - --symlink + - --target-dir=/opt/etcd-v3.5.0 + - --target-dir=/opt/etcd-v3.5.1 + - --target-dir=/opt/etcd-v3.5.3 + - --target-dir=/opt/etcd-v3.5.4 + - --target-dir=/opt/etcd-v3.5.6 + - --target-dir=/opt/etcd-v3.5.7 + - --target-dir=/opt/etcd-v3.5.9 + - --src=/opt/etcd-v3.5.13/etcd + - --src=/opt/etcd-v3.5.13/etcdctl + command: + - /opt/kops-utils/kops-utils-cp + image: registry.k8s.io/kops/kops-utils-cp:1.31.0-beta.1 + name: init-etcd-symlinks-3-5-13 + resources: {} + volumeMounts: + - mountPath: /opt + name: opt + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content new file mode 100644 index 0000000000000..21a36c9bb0d56 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null +spec: + containers: + - args: + - --ca-cert=/secrets/ca.crt + - --client-cert=/secrets/client.crt + - --client-key=/secrets/client.key + image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.31.0-beta.1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /.kube-apiserver-healthcheck/healthz + port: 3990 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: healthcheck + resources: {} + securityContext: + runAsNonRoot: true + runAsUser: 10012 + volumeMounts: + - mountPath: /secrets + name: healthcheck-secrets + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/kube-apiserver-healthcheck/secrets + type: Directory + name: healthcheck-secrets +status: {} diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content new file mode 100644 index 0000000000000..31d50e4d2beeb --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-master-us-test-1a_content @@ -0,0 +1,334 @@ +APIServerConfig: + API: + publicName: api.privatekindnet.example.com + ClusterDNSDomain: cluster.local + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: external + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-apiserver:v1.25.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.privatekindnet.example.com + serviceAccountJWKSURI: https://api.internal.privatekindnet.example.com/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANiW3hfHTcKnxCig+uWhpVbOfH1pANKm + XVSysPKgE80QSU4tZ6m49pAEeIMsvwvDMaLsb2v6JvXe0qvCmueU+/sCAwEAAQ== + -----END RSA PUBLIC KEY----- + -----BEGIN RSA PUBLIC KEY----- + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKOE64nZbH+GM91AIrqf7HEk4hvzqsZF + Ftxc+8xir1XC3mI/RhCCrs6AdVRZNZ26A6uHArhi33c2kHQkCjyLA7sCAwEAAQ== + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubelet,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz + - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 + - 71aee9d987b7fad0ff2ade50b038ad7e2356324edc02c54045960a3521b3e6a7@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-amd64.tar.gz + - d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz + - f90ed6dcef534e6d1ae17907dc7eb40614b8945ad4af7f0e98d2be7cde8165c6@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-amd64 + - 9992e7eb2a2e93f799e5a9e98eb718637433524bc65f630357201a79f49b13d0@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-amd64 + arm64: + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubelet,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubectl,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz + - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64 + - d8df47708ca57b9cd7f498055126ba7dcfc811d9ba43aae1830c93a09e70e22d@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-arm64.tar.gz + - 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-arm64.tar.gz + - 2f599c3d54f4c4bdbcc95aaf0c7b513a845d8f9503ec5b34c9f86aa1bc34fc0c@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/protokube-linux-arm64 + - 9d842e3636a95de2315cdea2be7a282355aac0658ef0b86d5dc2449066538f13@https://artifacts.k8s.io/binaries/kops/1.21.0-alpha.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.21.0-alpha.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gINaZLHjisEcbMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTExMloX + DTMxMDYzMDA0NTExMlowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQAHAomFKsF4jvYX + WM/UzQXDj9nSAFTf8dBPCXyZZNotsOH7+P6W4mMiuVs8bAuGiXGUdbsQ2lpiT/Rk + CzMeMdr4 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgjCCASygAwIBAgIMFo3gM0nxQpiX/agfMA0GCSqGSIb3DQEBCwUAMCIxIDAe + BgNVBAMTF2FwaXNlcnZlci1hZ2dyZWdhdG9yLWNhMB4XDTIxMDYzMDA0NTIzMVoX + DTMxMDYzMDA0NTIzMVowIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3It + Y2EwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAyyE71AOU3go5XFegLQ6fidI0LhhM + x7CzpTzh2xWKcHUfbNI7itgJvC/+GlyG5W+DF5V7ba0IJiQLsFve0oLdewIDAQAB + o0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU + ALfqF5ZmfqvqORuJIFilZYKF3d0wDQYJKoZIhvcNAQELBQADQQCXsoezoxXu2CEN + QdlXZOfmBT6cqxIX/RMHXhpHwRiqPsTO8IO2bVA8CSzxNwMuSv/ZtrMHoh8+PcVW + HLtkTXH8 + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1ogHnr26DL9YkqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjE5MDFaFw0zMTA2Mjgx + NjE5MDFaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAAZAdf8ROEVkr3Rf7I+s+CQOil2toadlKWOY + qCeJ2XaEROfp9aUTEIU1MGM3g57MPyAPPU7mURskuOQz6B1UFaY= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBcjCCARygAwIBAgIMFo1olfBnC/CsT+dqMA0GCSqGSIb3DQEBCwUAMBoxGDAW + BgNVBAMTD2V0Y2QtY2xpZW50cy1jYTAeFw0yMTA2MjgxNjIwMzNaFw0zMTA2Mjgx + NjIwMzNaMBoxGDAWBgNVBAMTD2V0Y2QtY2xpZW50cy1jYTBcMA0GCSqGSIb3DQEB + AQUAA0sAMEgCQQDYlt4Xx03Cp8QooPrloaVWznx9aQDSpl1UsrDyoBPNEElOLWep + uPaQBHiDLL8LwzGi7G9r+ib13tKrwprnlPv7AgMBAAGjQjBAMA4GA1UdDwEB/wQE + AwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQjlt4Ue54AbJPWlDpRM51s + x+PeBDANBgkqhkiG9w0BAQsFAANBAF1xUz77PlUVUnd9duF8F7plou0TONC9R6/E + YQ8C6vM1b+9NSDGjCW8YmwEU2fBgskb/BBX2lwVZ32/RUEju4Co= + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bKjm04vB4rNtaMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAwOTU2WhcN + MzEwNzA1MjAwOTU2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKiC8tndMlEFZ7qzeKxeKqFVjaYpsh/H + g7RxWo15+1kgH3suO0lxp9+RxSVv97hnsfbySTPZVhy2cIQj7eZtZt8CAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFBg6 + CEZkQNnRkARBwFce03AEWa+sMA0GCSqGSIb3DQEBCwUAA0EAJMnBThok/uUe8q8O + sS5q19KUuE8YCTUzMDj36EBKf6NX4NoakCa1h6kfQVtlMtEIMWQZCjbm8xGK5ffs + GS/VUw== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBgDCCASqgAwIBAgIMFo+bQ+EgIiBmGghjMA0GCSqGSIb3DQEBCwUAMCExHzAd + BgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1ldmVudHMwHhcNMjEwNzA1MjAxMTQ2WhcN + MzEwNzA1MjAxMTQ2WjAhMR8wHQYDVQQDExZldGNkLW1hbmFnZXItY2EtZXZlbnRz + MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKFhHVVxxDGv8d1jBvtdSxz7KIVoBOjL + DMxsmTsINiQkTQaFlb+XPlnY1ar4+RhE519AFUkqfhypk4Zxqf1YFXUCAwEAAaNC + MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNuW + LLH5c8kDubDbr6BHgedW0iJ9MA0GCSqGSIb3DQEBCwUAA0EAiKUoBoaGu7XzboFE + hjfKlX0TujqWuW3qMxDEJwj4dVzlSLrAoB/G01MJ+xxYKh456n48aG6N827UPXhV + cPfVNg== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjm1c3jfv6hIMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAxbkDbGYmCSShpRG3r+lzTOFujyuruRfjOhYm + ZRX4w1Utd5y63dUc98sjc9GGUYMHd+0k1ql/a48tGhnK6N6jJwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWZLkbBFx + GAgPU4i62c52unSo7RswDQYJKoZIhvcNAQELBQADQQAj6Pgd0va/8FtkyMlnohLu + Gf4v8RJO6zk3Y6jJ4+cwWziipFM1ielMzSOZfFcCZgH3m5Io40is4hPSqyq2TOA6 + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eg8Si30gr4MA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWluMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1tYW5hZ2VyLWNhLW1haW4wXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAw33jzcd/iosN04b0WXbDt7B0c3sJ3aafcGLP + vG3xRB9N5bYr9+qZAq3mzAFkxscn4j1ce5b1/GKTDEAClmZgdQIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUE/h+3gDP + DvKwHRyiYlXM8voZ1wowDQYJKoZIhvcNAQELBQADQQBXuimeEoAOu5HN4hG7NqL9 + t40K3ZRhRZv3JQWnRVJCBDjg1rD0GQJR/n+DoWvbeijI5C9pNjr2pWSIYR1eYCvd + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bKjmxTPh3/lYJMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMDk1NloXDTMx + MDcwNTIwMDk1NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAv5g4HF2xmrYyouJfY9jXx1M3gPLD/pupvxPY + xyjJw5pNCy5M5XGS3iTqRD5RDE0fWudVHFZKLIe8WPc06NApXwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUf6xiDI+O + Yph1ziCGr2hZaQYt+fUwDQYJKoZIhvcNAQELBQADQQBBxj5hqEQstonTb8lnqeGB + DEYtUeAk4eR/HzvUMjF52LVGuvN3XVt+JTrFeKNvb6/RDUbBNRj3azalcUkpPh6V + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBfDCCASagAwIBAgIMFo+bQ+Eq69jgzpKwMA0GCSqGSIb3DQEBCwUAMB8xHTAb + BgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRzMB4XDTIxMDcwNTIwMTE0NloXDTMx + MDcwNTIwMTE0NlowHzEdMBsGA1UEAxMUZXRjZC1wZWVycy1jYS1ldmVudHMwXDAN + BgkqhkiG9w0BAQEFAANLADBIAkEAo5Nj2CjX1qp3mEPw1H5nHAFWLoGNSLSlRFJW + 03NxaNPMFzL5PrCoyOXrX8/MWczuZYw0Crf8EPOOQWi2+W0XLwIDAQABo0IwQDAO + BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUxauhhKQh + cvdZND78rHe0RQVTTiswDQYJKoZIhvcNAQELBQADQQB+cq4jIS9q0zXslaRa+ViI + J+dviA3sMygbmSJO0s4DxYmoazKJblux5q0ASSvS9iL1l9ShuZ1dWyp2tpZawHyb + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bKjmuLDDLcDHsMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDA5NTZaFw0zMTA3 + MDUyMDA5NTZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCyRaXWpwgN6INQqws9p/BvPElJv2Rno9dVTFhlQqDA + aUJXe7MBmiO4NJcW76EozeBh5ztR3/4NE1FM2x8TisS3AgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBQtE1d49uSvpURf + OQ25Vlu6liY20DANBgkqhkiG9w0BAQsFAANBAAgLVaetJZcfOA3OIMMvQbz2Ydrt + uWF9BKkIad8jrcIrm3IkOtR8bKGmDIIaRKuG/ZUOL6NMe2fky3AAfKwleL4= + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBeDCCASKgAwIBAgIMFo+bQ+EuVthBfuZvMA0GCSqGSIb3DQEBCwUAMB0xGzAZ + BgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjAeFw0yMTA3MDUyMDExNDZaFw0zMTA3 + MDUyMDExNDZaMB0xGzAZBgNVBAMTEmV0Y2QtcGVlcnMtY2EtbWFpbjBcMA0GCSqG + SIb3DQEBAQUAA0sAMEgCQQCxNbycDZNx5V1ZOiXxZSvaFpHRwKeHDfcuMUitdoPt + naVMlMTGDWAMuCVmFHFAWohIYynemEegmZkZ15S7AErfAgMBAAGjQjBAMA4GA1Ud + DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTAjQ8T4HclPIsC + qipEfUIcLP6jqTANBgkqhkiG9w0BAQsFAANBAJdZ17TN3HlWrH7HQgfR12UBwz8K + G9DurDznVaBVUYaHY8Sg5AvAXeb+yIF2JMmRR+bK+/G1QYY2D3/P31Ic2Oo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw + ODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1 + jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA + MA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8 + tmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w== + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + MIIBbjCCARigAwIBAgIMFpANvmSa0OAlYmXKMA0GCSqGSIb3DQEBCwUAMBgxFjAU + BgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwOTM2WhcNMzEwNzA3MDcw + OTM2WjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD + SwAwSAJBAMF6F4aZdpe0RUpyykaBpWwZCnwbffhYGOw+fs6RdLuUq7QCNmJm/Eq7 + WWOziMYDiI9SbclpD+6QiJ0N3EqppVUCAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG + MA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFLImp6ARjPDAH6nhI+scWVt3Q9bn + MA0GCSqGSIb3DQEBCwUAA0EAVQVx5MUtuAIeePuP9o51xtpT2S6Fvfi8J4ICxnlA + 9B7UD2ushcVFPtaeoL9Gfu8aY4KJBeqqg5ojl4qmRnThjw== + -----END CERTIFICATE----- +ClusterName: privatekindnet.example.com +ControlPlaneConfig: + KubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: privatekindnet.example.com + configureCloudRoutes: false + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-controller-manager:v1.25.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + KubeScheduler: + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + image: registry.k8s.io/kube-scheduler:v1.25.0 + leaderElection: + leaderElect: true + logLevel: 2 +DNSZone: Z1AFAKE1ZON3YO +EtcdClusterNames: +- main +- events +FileAssets: +- content: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration + path: /var/lib/kube-scheduler/config.yaml +Hooks: +- null +- null +KeypairIDs: + apiserver-aggregator-ca: "6980187172486667078076483355" + etcd-clients-ca: "6979622252718071085282986282" + etcd-manager-ca-events: "6982279354000777253151890266" + etcd-manager-ca-main: "6982279354000936168671127624" + etcd-peers-ca-events: "6982279353999767935825892873" + etcd-peers-ca-main: "6982279353998887468930183660" + kubernetes-ca: "6982820025135291416230495506" + service-account: "2" +KubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.25.0 + logLevel: 2 +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/control-plane: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + taints: + - node-role.kubernetes.io/control-plane=:NoSchedule +KubernetesVersion: 1.25.0 +Networking: + kindnet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://clusters.example.com/privatekindnet.example.com/addons/bootstrap-channel.yaml +configStore: + keypairs: memfs://clusters.example.com/privatekindnet.example.com/pki + secrets: memfs://clusters.example.com/privatekindnet.example.com/secrets +containerdConfig: + logLevel: info + runc: + version: 1.1.5 + version: 1.6.20 +etcdManifests: +- memfs://clusters.example.com/privatekindnet.example.com/manifests/etcd/main-master-us-test-1a.yaml +- memfs://clusters.example.com/privatekindnet.example.com/manifests/etcd/events-master-us-test-1a.yaml +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml +usesLegacyGossip: false +usesNoneDNS: false diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content new file mode 100644 index 0000000000000..9e9148cfb6dd3 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_nodeupconfig-nodes_content @@ -0,0 +1,64 @@ +Assets: + amd64: + - 7f9183fce12606818612ce80b6c09757452c4fb50aefea5fc5843951c5020e24@https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubelet,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubelet + - e23cc7092218c95c22d8ee36fb9499194a36ac5b5349ca476886b7edc0203885@https://dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/amd64/kubectl + - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz + - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz + - f00b144e86f8c1db347a2e8f22caade07d55382c5f76dd5c0a5b1ab64eaec8bb@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 + - 71aee9d987b7fad0ff2ade50b038ad7e2356324edc02c54045960a3521b3e6a7@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-amd64.tar.gz + - d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz + arm64: + - 69572a7b3d179d4a479aa2e0f90e2f091d8d84ef33a35422fc89975dc137a590@https://dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubelet,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubelet + - 24db547bbae294c5c44f2b4a777e45f0e2f3d6295eace0d0c4be2b2dfa45330d@https://dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubectl,https://cdn.dl.k8s.io/release/v1.25.0/bin/linux/arm64/kubectl + - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz + - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz + - 54e79e4d48b9e191767e4abc08be1a8476a1c757e9a9f8c45c6ded001226867f@https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.arm64 + - d8df47708ca57b9cd7f498055126ba7dcfc811d9ba43aae1830c93a09e70e22d@https://github.com/containerd/nerdctl/releases/download/v1.7.4/nerdctl-1.7.4-linux-arm64.tar.gz + - 0b615cfa00c331fb9c4524f3d4058a61cc487b33a3436d1269e7832cf283f925@https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-arm64.tar.gz +CAs: {} +ClusterName: privatekindnet.example.com +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "6982820025135291416230495506" +KubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + image: registry.k8s.io/kube-proxy:v1.25.0 + logLevel: 2 +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + CSIMigrationAWS: "true" + InTreePluginAWSUnregister: "true" + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + node-role.kubernetes.io/node: "" + podInfraContainerImage: registry.k8s.io/pause:3.9 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + registerSchedulable: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +KubernetesVersion: 1.25.0 +Networking: + kindnet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +containerdConfig: + logLevel: info + runc: + version: 1.1.5 + version: 1.6.20 +usesLegacyGossip: false +usesNoneDNS: false diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content new file mode 100644 index 0000000000000..e233f0148ed8e --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content @@ -0,0 +1,237 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + k8s-app: aws-cloud-controller-manager + name: aws-cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + template: + metadata: + creationTimestamp: null + labels: + k8s-app: aws-cloud-controller-manager + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --allocate-node-cidrs=true + - --cluster-cidr=100.96.0.0/11 + - --cluster-name=privatekindnet.example.com + - --configure-cloud-routes=false + - --leader-elect=true + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --cloud-config=/etc/kubernetes/cloud.config + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.25.15 + imagePullPolicy: IfNotPresent + name: aws-cloud-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccountName: aws-cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + type: "" + name: cloudconfig + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resourceNames: + - node-controller + - service-controller + - route-controller + resources: + - serviceaccounts/token + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: aws-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content new file mode 100644 index 0000000000000..78ac12d3d37f3 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content @@ -0,0 +1,1122 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + +--- + +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-role +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - patch + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattributesclasses + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattributesclasses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-external-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node-getter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node-role +subjects: +- kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-leases-role + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-leases-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ebs-csi-leases-role +subjects: +- kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app: ebs-csi-controller + app.kubernetes.io/managed-by: kops + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + ports: + - name: metrics + port: 3301 + targetPort: 3301 + selector: + app: ebs-csi-controller + type: ClusterIP + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-node + namespace: kube-system +spec: + revisionHistoryLimit: 10 + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-node + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: Exists + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: node.kubernetes.io/instance-type + operator: NotIn + values: + - a1.medium + - a1.large + - a1.xlarge + - a1.2xlarge + - a1.4xlarge + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logging-format=text + - --v=2 + env: + - name: AWS_REGION + value: us-test-1 + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /bin/aws-ebs-csi-driver + - pre-stop-hook + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + privileged: true + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar:v2.11.0-eks-1-30-10 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 + name: node-driver-registrar + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - mountPath: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + name: probe-dir + - args: + - --csi-address=/csi/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.13.0-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: plugin-dir + hostNetwork: false + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + securityContext: + fsGroup: 0 + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + serviceAccountName: ebs-csi-node-sa + terminationGracePeriodSeconds: 30 + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + - emptyDir: {} + name: probe-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: ebs-csi-controller + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + weight: 1 + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - ebs-csi-controller + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - controller + - --endpoint=$(CSI_ENDPOINT) + - --k8s-tag-cluster-id=privatekindnet.example.com + - --extra-tags=KubernetesCluster=privatekindnet.example.com + - --http-endpoint=0.0.0.0:3301 + - --batching=true + - --logging-format=text + - --v=5 + env: + - name: AWS_REGION + value: us-test-1 + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + - name: AWS_EC2_ENDPOINT + valueFrom: + configMapKeyRef: + key: endpoint + name: aws-meta + optional: true + image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9811 + name: healthz + protocol: TCP + - containerPort: 3301 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + - --retry-interval-max=30m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner:v5.0.1-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: csi-provisioner + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + - --retry-interval-max=5m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher:v4.6.1-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: csi-attacher + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s:v0.3.0 + imagePullPolicy: IfNotPresent + name: volumemodifier + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --timeout=60s + - --csi-address=$(ADDRESS) + - --v=5 + - --handle-volume-inuse-error=false + - --leader-election=true + - --kube-api-qps=20 + - --kube-api-burst=100 + - --workers=100 + - --retry-interval-max=30m + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer:v1.11.1-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: csi-resizer + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + - --http-endpoint=0.0.0.0:9811 + image: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe:v2.13.0-eks-1-30-10 + imagePullPolicy: IfNotPresent + name: liveness-probe + resources: + limits: + memory: 256Mi + requests: + cpu: 10m + memory: 40Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /csi + name: socket-dir + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: ebs-csi-controller-sa + tolerations: + - operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/name: aws-ebs-csi-driver + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - emptyDir: {} + name: socket-dir + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: aws-ebs-csi-driver.addons.k8s.io + app.kubernetes.io/component: csi-driver + app.kubernetes.io/instance: aws-ebs-csi-driver + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-ebs-csi-driver + app.kubernetes.io/version: v1.33.0 + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content new file mode 100644 index 0000000000000..9c9cea3663d22 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content @@ -0,0 +1,128 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: ea8ab18257d4a4961926a6a0ff3dacfb095c6ebfe71b695cfc32688ca0765c43 + name: kops-controller.addons.k8s.io + needsRollingUpdate: control-plane + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: coredns.addons.k8s.io/k8s-1.12.yaml + manifestHash: 591e3b40d00949575616698ce1c9230db8cb00bdab4f8a0d5ef14080a1d7a93c + name: coredns.addons.k8s.io + selector: + k8s-addon: coredns.addons.k8s.io + version: 9.99.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: 01c120e887bd98d82ef57983ad58a0b22bc85efb48108092a24c4b82e4c9ea81 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: 9.99.0 + - id: k8s-1.23 + manifest: leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml + manifestHash: b9c91e09c0f28c9b74ff140b8395d611834c627d698846d625c10975a74a48c4 + name: leader-migration.rbac.addons.k8s.io + selector: + k8s-addon: leader-migration.rbac.addons.k8s.io + version: 9.99.0 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2d55c3bc5e354e84a3730a65b42f39aba630a59dc8d32b30859fcce3d3178bc2 + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: 7a1b4740d79bc35f7fe0018fdd192743ef961feae4c065714dd6b59767125deb + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.11 + manifest: node-termination-handler.aws/k8s-1.11.yaml + manifestHash: e368ee43ac848123dc417eb01da535dd834120490be6456dcf9f1c58817f1cd3 + name: node-termination-handler.aws + prune: + kinds: + - kind: ConfigMap + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - kind: Service + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - kind: ServiceAccount + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: apps + kind: DaemonSet + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: apps + kind: Deployment + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: apps + kind: StatefulSet + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: policy + kind: PodDisruptionBudget + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: rbac.authorization.k8s.io + kind: ClusterRole + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: Role + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: RoleBinding + labelSelector: addon.kops.k8s.io/name=node-termination-handler.aws,app.kubernetes.io/managed-by=kops + selector: + k8s-addon: node-termination-handler.aws + version: 9.99.0 + - id: v1.15.0 + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 4e2cda50cd5048133aad1b5e28becb60f4629d3f9e09c514a2757c27998b4200 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: networking.kindnet/k8s-1.12.yaml + manifestHash: edf14069db8b954139810f47ef0acb654c6e1e520a9f87d41cab0da72206ada1 + name: networking.kindnet + needsRollingUpdate: all + selector: + role.kubernetes.io/networking: "1" + version: 9.99.0 + - id: k8s-1.18 + manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml + manifestHash: 919559e154f56760f00d4532e46addc09ac6f869e4f631fa62995a7ed90a7121 + name: aws-cloud-controller.addons.k8s.io + selector: + k8s-addon: aws-cloud-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.17 + manifest: aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml + manifestHash: 0c2d807c601dcfc092c270e3c338a9245dd09bceee62b2e5697f7b45dd627e7b + name: aws-ebs-csi-driver.addons.k8s.io + selector: + k8s-addon: aws-ebs-csi-driver.addons.k8s.io + version: 9.99.0 diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000000..31d86455d155c --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,383 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: coredns + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system + +--- + +apiVersion: v1 +data: + Corefile: |- + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local. in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + addonmanager.kubernetes.io/mode: EnsureExists + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: registry.k8s.io/coredns/coredns:v1.11.3 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + volumes: + - configMap: + name: coredns + name: config-volume + +--- + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: kube-dns + namespace: kube-system + resourceVersion: "0" +spec: + clusterIP: 100.64.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + selector: + k8s-app: kube-dns + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: kube-dns + namespace: kube-system +spec: + maxUnavailable: 50% + selector: + matchLabels: + k8s-app: kube-dns + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - replicationcontrollers/scale + verbs: + - get + - update +- apiGroups: + - extensions + - apps + resources: + - deployments/scale + - replicasets/scale + verbs: + - get + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coredns-autoscaler +subjects: +- kind: ServiceAccount + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: coredns-autoscaler + kubernetes.io/cluster-service: "true" + name: coredns-autoscaler + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: coredns-autoscaler + template: + metadata: + creationTimestamp: null + labels: + k8s-app: coredns-autoscaler + kops.k8s.io/managed-by: kops + spec: + containers: + - command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=coredns-autoscaler + - --target=Deployment/coredns + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} + - --logtostderr=true + - --v=2 + image: registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.8.9 + name: autoscaler + resources: + requests: + cpu: 20m + memory: 10Mi + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns-autoscaler + tolerations: + - key: CriticalAddonsOnly + operator: Exists diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000000..9c176638a5d93 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,138 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.31.0-beta.1 + name: dns-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + strategy: + type: Recreate + template: + metadata: + creationTimestamp: null + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + kops.k8s.io/managed-by: kops + version: v1.31.0-beta.1 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --watch-ingress=false + - --dns=aws-route53 + - --zone=*/Z1AFAKE1ZON3YO + - --internal-ipv4 + - --zone=*/* + - -v=2 + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/kops/dns-controller:1.31.0-beta.1 + name: dns-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: dns-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: dns-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - ingress + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: dns-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:dns-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:dns-controller diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content new file mode 100644 index 0000000000000..9ba75c322e769 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -0,0 +1,225 @@ +apiVersion: v1 +data: + config.yaml: | + {"clusterName":"privatekindnet.example.com","cloud":"aws","configBase":"memfs://clusters.example.com/privatekindnet.example.com","secretStore":"memfs://clusters.example.com/privatekindnet.example.com/secrets","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.privatekindnet.example.com"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}} +kind: ConfigMap +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.31.0-beta.1 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + annotations: + dns.alpha.kubernetes.io/internal: kops-controller.internal.privatekindnet.example.com + creationTimestamp: null + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + kops.k8s.io/managed-by: kops + version: v1.31.0-beta.1 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + containers: + - args: + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/kops/kops-controller:1.31.0-beta.1 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + runAsUser: 10011 + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: kops-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + type: OnDelete + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + - coordination.k8s.io + resourceNames: + - kops-controller-leader + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content new file mode 100644 index 0000000000000..36761e1c56255 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kubelet-api.rbac.addons.k8s.io + name: kops:system:kubelet-api-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubelet-api diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000000..11ed6d46fd241 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,52 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +rules: +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-provider-extraction-migration + resources: + - leases + verbs: + - create + - list + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: leader-migration.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: leader-migration.rbac.addons.k8s.io + name: system::leader-locking-migration + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-migration +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:kube-controller-manager +- kind: ServiceAccount + name: kube-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: aws-cloud-controller-manager + namespace: kube-system +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content new file mode 100644 index 0000000000000..4dcdce48b9ab9 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: LimitRange +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: limit-range.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: limit-range.addons.k8s.io + name: limits + namespace: default +spec: + limits: + - defaultRequest: + cpu: 100m + type: Container diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.12_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.12_content new file mode 100644 index 0000000000000..990bebfe74590 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.12_content @@ -0,0 +1,167 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: kindnet +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - patch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: +- kind: ServiceAccount + name: kindnet + namespace: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: kindnet + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: networking.kindnet + app: kindnet + app.kubernetes.io/managed-by: kops + k8s-app: kindnet + role.kubernetes.io/networking: "1" + tier: node + name: kindnet + namespace: kube-system +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + creationTimestamp: null + labels: + app: kindnet + k8s-app: kindnet + kops.k8s.io/managed-by: kops + tier: node + spec: + containers: + - args: + - /bin/kindnetd + - --hostname-override=$(NODE_NAME) + - --v=2 + - --masquerading=true + - --no-masquerade-cidr=172.20.0.0/16,100.96.0.0/11,100.64.0.0/13 + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: ghcr.io/aojea/kindnetd:stable + name: kindnet-cni + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + capabilities: + add: + - NET_RAW + - NET_ADMIN + privileged: false + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni-cfg + - mountPath: /var/lib/cni-kindnet + name: var-lib-kindnet + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + initContainers: + - command: + - sh + - -c + - cat /opt/cni/bin/cni-kindnet > /cni/cni-kindnet ; chmod +x /cni/cni-kindnet + image: ghcr.io/aojea/kindnetd:stable + name: install-cni-bin + volumeMounts: + - mountPath: /cni + name: cni-bin + serviceAccountName: kindnet + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-bin + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: cni-cfg + - hostPath: + path: /var/lib/cni-kindnet + type: DirectoryOrCreate + name: var-lib-kindnet diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content new file mode 100644 index 0000000000000..12dee2372a19a --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content @@ -0,0 +1,285 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + k8s-app: aws-node-termination-handler + name: aws-node-termination-handler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + name: aws-node-termination-handler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - extensions + resources: + - daemonsets + verbs: + - get +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + name: aws-node-termination-handler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node-termination-handler +subjects: +- kind: ServiceAccount + name: aws-node-termination-handler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/component: deployment + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + app.kubernetes.io/part-of: aws-node-termination-handler + app.kubernetes.io/version: v1.22.0 + k8s-addon: node-termination-handler.aws + k8s-app: aws-node-termination-handler + name: aws-node-termination-handler + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kubernetes.io/os: linux + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: deployment + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + k8s-app: aws-node-termination-handler + kops.k8s.io/managed-by: kops + kops.k8s.io/nth-mode: sqs + kubernetes.io/os: linux + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_PROBES_SERVER + value: "true" + - name: PROBES_SERVER_PORT + value: "8080" + - name: PROBES_SERVER_ENDPOINT + value: /healthz + - name: LOG_LEVEL + value: info + - name: JSON_LOGGING + value: "true" + - name: LOG_FORMAT_VERSION + value: "2" + - name: ENABLE_PROMETHEUS_SERVER + value: "false" + - name: PROMETHEUS_SERVER_PORT + value: "9092" + - name: CHECK_TAG_BEFORE_DRAINING + value: "true" + - name: MANAGED_TAG + value: aws-node-termination-handler/managed + - name: USE_PROVIDER_ID + value: "true" + - name: DRY_RUN + value: "false" + - name: CORDON_ONLY + value: "false" + - name: TAINT_NODE + value: "false" + - name: EXCLUDE_FROM_LOAD_BALANCERS + value: "true" + - name: DELETE_LOCAL_DATA + value: "true" + - name: IGNORE_DAEMON_SETS + value: "true" + - name: POD_TERMINATION_GRACE_PERIOD + value: "-1" + - name: NODE_TERMINATION_GRACE_PERIOD + value: "120" + - name: EMIT_KUBERNETES_EVENTS + value: "true" + - name: COMPLETE_LIFECYCLE_ACTION_DELAY_SECONDS + value: "-1" + - name: ENABLE_SQS_TERMINATION_DRAINING + value: "true" + - name: QUEUE_URL + value: https://sqs.us-test-1.amazonaws.com/123456789012/privatekindnet-example-com-nth + - name: DELETE_SQS_MSG_IF_NODE_NOT_FOUND + value: "false" + - name: WORKERS + value: "10" + image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.22.0 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + name: aws-node-termination-handler + ports: + - containerPort: 8080 + name: liveness-probe + protocol: TCP + - containerPort: 9092 + name: metrics + protocol: TCP + resources: + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + securityContext: + fsGroup: 1000 + serviceAccountName: aws-node-termination-handler + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kops.k8s.io/nth-mode: sqs + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kops.k8s.io/nth-mode: sqs + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: node-termination-handler.aws + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: aws-node-termination-handler + k8s-addon: node-termination-handler.aws + name: aws-node-termination-handler + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: aws-node-termination-handler + app.kubernetes.io/name: aws-node-termination-handler + kops.k8s.io/nth-mode: sqs diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content new file mode 100644 index 0000000000000..bea3e88be3ce7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content @@ -0,0 +1,118 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: default +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: gp2 +parameters: + type: gp2 +provisioner: kubernetes.io/aws-ebs + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-ssd-1-17 +parameters: + encrypted: "true" + type: gp2 +provisioner: kubernetes.io/aws-ebs +volumeBindingMode: WaitForFirstConsumer + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: kops-csi-1-21 +parameters: + encrypted: "true" + type: gp3 +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + labels: + addon.kops.k8s.io/name: storage-aws.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: storage-aws.addons.k8s.io + name: system:aws-cloud-provider +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:aws-cloud-provider +subjects: +- kind: ServiceAccount + name: aws-cloud-provider + namespace: kube-system diff --git a/tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy b/tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy new file mode 100644 index 0000000000000..5acbcace3b129 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/data/aws_sqs_queue_privatekindnet-example-com-nth_policy @@ -0,0 +1,16 @@ +{ + "Statement": [ + { + "Action": "sqs:SendMessage", + "Effect": "Allow", + "Principal": { + "Service": [ + "events.amazonaws.com", + "sqs.amazonaws.com" + ] + }, + "Resource": "arn:aws-test:sqs:us-test-1:123456789012:privatekindnet-example-com-nth" + } + ], + "Version": "2012-10-17" +} diff --git a/tests/integration/update_cluster/privatekindnet/id_rsa.pub b/tests/integration/update_cluster/privatekindnet/id_rsa.pub new file mode 100644 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml b/tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml new file mode 100644 index 0000000000000..1ce4d63d4b3f4 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/in-v1alpha2.yaml @@ -0,0 +1,98 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-12T04:13:14Z" + name: privatekindnet.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/privatekindnet.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.25.0 + masterPublicName: api.privatekindnet.example.com + networkCIDR: 172.20.0.0/16 + networking: + kindnet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Private + zone: us-test-1a + - cidr: 172.20.4.0/22 + name: utility-us-test-1a + type: Utility + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: privatekindnet.example.com +spec: + associatePublicIp: true + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-12T04:13:15Z" + name: nodes + labels: + kops.k8s.io/cluster: privatekindnet.example.com +spec: + associatePublicIp: true + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-14T15:32:41Z" + name: bastion + labels: + kops.k8s.io/cluster: privatekindnet.example.com +spec: + associatePublicIp: true + image: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220404 + machineType: t2.micro + maxSize: 1 + minSize: 1 + role: Bastion + subnets: + - utility-us-test-1a diff --git a/tests/integration/update_cluster/privatekindnet/kubernetes.tf b/tests/integration/update_cluster/privatekindnet/kubernetes.tf new file mode 100644 index 0000000000000..76e123e92e2b0 --- /dev/null +++ b/tests/integration/update_cluster/privatekindnet/kubernetes.tf @@ -0,0 +1,1475 @@ +locals { + bastion_autoscaling_group_ids = [aws_autoscaling_group.bastion-privatekindnet-example-com.id] + bastion_security_group_ids = [aws_security_group.bastion-privatekindnet-example-com.id] + bastions_role_arn = aws_iam_role.bastions-privatekindnet-example-com.arn + bastions_role_name = aws_iam_role.bastions-privatekindnet-example-com.name + cluster_name = "privatekindnet.example.com" + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-privatekindnet-example-com.id] + master_security_group_ids = [aws_security_group.masters-privatekindnet-example-com.id] + masters_role_arn = aws_iam_role.masters-privatekindnet-example-com.arn + masters_role_name = aws_iam_role.masters-privatekindnet-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-privatekindnet-example-com.id] + node_security_group_ids = [aws_security_group.nodes-privatekindnet-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-privatekindnet-example-com.id] + nodes_role_arn = aws_iam_role.nodes-privatekindnet-example-com.arn + nodes_role_name = aws_iam_role.nodes-privatekindnet-example-com.name + region = "us-test-1" + route_table_private-us-test-1a_id = aws_route_table.private-us-test-1a-privatekindnet-example-com.id + route_table_public_id = aws_route_table.privatekindnet-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-privatekindnet-example-com.id + subnet_utility-us-test-1a_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id + vpc_cidr_block = aws_vpc.privatekindnet-example-com.cidr_block + vpc_id = aws_vpc.privatekindnet-example-com.id + vpc_ipv6_cidr_block = aws_vpc.privatekindnet-example-com.ipv6_cidr_block + vpc_ipv6_cidr_length = local.vpc_ipv6_cidr_block == "" ? null : tonumber(regex(".*/(\\d+)", local.vpc_ipv6_cidr_block)[0]) +} + +output "bastion_autoscaling_group_ids" { + value = [aws_autoscaling_group.bastion-privatekindnet-example-com.id] +} + +output "bastion_security_group_ids" { + value = [aws_security_group.bastion-privatekindnet-example-com.id] +} + +output "bastions_role_arn" { + value = aws_iam_role.bastions-privatekindnet-example-com.arn +} + +output "bastions_role_name" { + value = aws_iam_role.bastions-privatekindnet-example-com.name +} + +output "cluster_name" { + value = "privatekindnet.example.com" +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-privatekindnet-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-privatekindnet-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-privatekindnet-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-privatekindnet-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-privatekindnet-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-privatekindnet-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-privatekindnet-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-privatekindnet-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-privatekindnet-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_private-us-test-1a_id" { + value = aws_route_table.private-us-test-1a-privatekindnet-example-com.id +} + +output "route_table_public_id" { + value = aws_route_table.privatekindnet-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-privatekindnet-example-com.id +} + +output "subnet_utility-us-test-1a_id" { + value = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.privatekindnet-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.privatekindnet-example-com.id +} + +output "vpc_ipv6_cidr_block" { + value = aws_vpc.privatekindnet-example-com.ipv6_cidr_block +} + +output "vpc_ipv6_cidr_length" { + value = local.vpc_ipv6_cidr_block == "" ? null : tonumber(regex(".*/(\\d+)", local.vpc_ipv6_cidr_block)[0]) +} + +provider "aws" { + region = "us-test-1" +} + +provider "aws" { + alias = "files" + region = "us-test-1" +} + +resource "aws_autoscaling_group" "bastion-privatekindnet-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.bastion-privatekindnet-example-com.id + version = aws_launch_template.bastion-privatekindnet-example-com.latest_version + } + max_instance_lifetime = 0 + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "bastion.privatekindnet.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "privatekindnet.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "bastion.privatekindnet.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/bastion" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "bastion" + } + tag { + key = "kubernetes.io/cluster/privatekindnet.example.com" + propagate_at_launch = true + value = "owned" + } + target_group_arns = [aws_lb_target_group.bastion-privatekindnet-ex-70mv5p.id] + vpc_zone_identifier = [aws_subnet.utility-us-test-1a-privatekindnet-example-com.id] +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-privatekindnet-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-privatekindnet-example-com.id + version = aws_launch_template.master-us-test-1a-masters-privatekindnet-example-com.latest_version + } + load_balancers = [aws_elb.api-privatekindnet-example-com.id] + max_instance_lifetime = 0 + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.privatekindnet.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "privatekindnet.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.privatekindnet.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/control-plane" + propagate_at_launch = true + value = "1" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/privatekindnet.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-privatekindnet-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-privatekindnet-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-privatekindnet-example-com.id + version = aws_launch_template.nodes-privatekindnet-example-com.latest_version + } + max_instance_lifetime = 0 + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "nodes.privatekindnet.example.com" + protect_from_scale_in = false + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "privatekindnet.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.privatekindnet.example.com" + } + tag { + key = "aws-node-termination-handler/managed" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" + propagate_at_launch = true + value = "" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/privatekindnet.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-privatekindnet-example-com.id] +} + +resource "aws_autoscaling_lifecycle_hook" "bastion-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.bastion-privatekindnet-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "bastion-NTHLifecycleHook" +} + +resource "aws_autoscaling_lifecycle_hook" "master-us-test-1a-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.master-us-test-1a-masters-privatekindnet-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "master-us-test-1a-NTHLifecycleHook" +} + +resource "aws_autoscaling_lifecycle_hook" "nodes-NTHLifecycleHook" { + autoscaling_group_name = aws_autoscaling_group.nodes-privatekindnet-example-com.id + default_result = "CONTINUE" + heartbeat_timeout = 300 + lifecycle_transition = "autoscaling:EC2_INSTANCE_TERMINATING" + name = "nodes-NTHLifecycleHook" +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-ASGLifecycle" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-ASGLifecycle_event_pattern") + name = "privatekindnet.example.com-ASGLifecycle" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-ASGLifecycle" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-InstanceScheduledChange" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceScheduledChange_event_pattern") + name = "privatekindnet.example.com-InstanceScheduledChange" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-InstanceScheduledChange" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-InstanceStateChange" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-InstanceStateChange_event_pattern") + name = "privatekindnet.example.com-InstanceStateChange" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-InstanceStateChange" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_rule" "privatekindnet-example-com-SpotInterruption" { + event_pattern = file("${path.module}/data/aws_cloudwatch_event_rule_privatekindnet.example.com-SpotInterruption_event_pattern") + name = "privatekindnet.example.com-SpotInterruption" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com-SpotInterruption" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-ASGLifecycle-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-ASGLifecycle.id +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-InstanceScheduledChange-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-InstanceScheduledChange.id +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-InstanceStateChange-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-InstanceStateChange.id +} + +resource "aws_cloudwatch_event_target" "privatekindnet-example-com-SpotInterruption-Target" { + arn = aws_sqs_queue.privatekindnet-example-com-nth.arn + rule = aws_cloudwatch_event_rule.privatekindnet-example-com-SpotInterruption.id +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-privatekindnet-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.etcd-events.privatekindnet.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-privatekindnet-example-com" { + availability_zone = "us-test-1a" + encrypted = false + iops = 3000 + size = 20 + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.etcd-main.privatekindnet.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + throughput = 125 + type = "gp3" +} + +resource "aws_eip" "us-test-1a-privatekindnet-example-com" { + domain = "vpc" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_elb" "api-privatekindnet-example-com" { + connection_draining = true + connection_draining_timeout = 300 + cross_zone_load_balancing = false + health_check { + healthy_threshold = 2 + interval = 10 + target = "SSL:443" + timeout = 5 + unhealthy_threshold = 2 + } + idle_timeout = 300 + listener { + instance_port = 443 + instance_protocol = "TCP" + lb_port = 443 + lb_protocol = "TCP" + } + name = "api-privatekindnet-exampl-c5d4tf" + security_groups = [aws_security_group.api-elb-privatekindnet-example-com.id] + subnets = [aws_subnet.utility-us-test-1a-privatekindnet-example-com.id] + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "api.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "bastions-privatekindnet-example-com" { + name = "bastions.privatekindnet.example.com" + role = aws_iam_role.bastions-privatekindnet-example-com.name + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastions.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "masters-privatekindnet-example-com" { + name = "masters.privatekindnet.example.com" + role = aws_iam_role.masters-privatekindnet-example-com.name + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "masters.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_instance_profile" "nodes-privatekindnet-example-com" { + name = "nodes.privatekindnet.example.com" + role = aws_iam_role.nodes-privatekindnet-example-com.name + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role" "bastions-privatekindnet-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_bastions.privatekindnet.example.com_policy") + name = "bastions.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastions.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role" "masters-privatekindnet-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.privatekindnet.example.com_policy") + name = "masters.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "masters.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role" "nodes-privatekindnet-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.privatekindnet.example.com_policy") + name = "nodes.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_iam_role_policy" "bastions-privatekindnet-example-com" { + name = "bastions.privatekindnet.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_bastions.privatekindnet.example.com_policy") + role = aws_iam_role.bastions-privatekindnet-example-com.name +} + +resource "aws_iam_role_policy" "masters-privatekindnet-example-com" { + name = "masters.privatekindnet.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.privatekindnet.example.com_policy") + role = aws_iam_role.masters-privatekindnet-example-com.name +} + +resource "aws_iam_role_policy" "nodes-privatekindnet-example-com" { + name = "nodes.privatekindnet.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.privatekindnet.example.com_policy") + role = aws_iam_role.nodes-privatekindnet-example-com.name +} + +resource "aws_internet_gateway" "privatekindnet-example-com" { + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_key_pair" "kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.privatekindnet.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.privatekindnet.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_launch_template" "bastion-privatekindnet-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 32 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.bastions-privatekindnet-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.micro" + key_name = aws_key_pair.kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + monitoring { + enabled = false + } + name = "bastion.privatekindnet.example.com" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.bastion-privatekindnet-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/role/bastion" = "1" + "kops.k8s.io/instancegroup" = "bastion" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/role/bastion" = "1" + "kops.k8s.io/instancegroup" = "bastion" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/role/bastion" = "1" + "kops.k8s.io/instancegroup" = "bastion" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-privatekindnet-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 64 + volume_type = "gp3" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-privatekindnet-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + monitoring { + enabled = false + } + name = "master-us-test-1a.masters.privatekindnet.example.com" + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.masters-privatekindnet-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "master-us-test-1a.masters.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "master-us-test-1a.masters.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "master-us-test-1a.masters.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/kops.k8s.io/kops-controller-pki" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/control-plane" = "" + "k8s.io/cluster-autoscaler/node-template/label/node.kubernetes.io/exclude-from-external-load-balancers" = "" + "k8s.io/role/control-plane" = "1" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.privatekindnet.example.com_user_data") +} + +resource "aws_launch_template" "nodes-privatekindnet-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + encrypted = true + iops = 3000 + throughput = 125 + volume_size = 128 + volume_type = "gp3" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-privatekindnet-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-privatekindnet-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + metadata_options { + http_endpoint = "enabled" + http_protocol_ipv6 = "disabled" + http_put_response_hop_limit = 1 + http_tokens = "optional" + } + monitoring { + enabled = false + } + name = "nodes.privatekindnet.example.com" + network_interfaces { + associate_public_ip_address = false + delete_on_termination = true + ipv6_address_count = 0 + security_groups = [aws_security_group.nodes-privatekindnet-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "aws-node-termination-handler/managed" = "" + "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node" = "" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.privatekindnet.example.com_user_data") +} + +resource "aws_lb" "bastion-privatekindnet-example-com" { + enable_cross_zone_load_balancing = false + internal = false + load_balancer_type = "network" + name = "bastion-privatekindnet-ex-70mv5p" + security_groups = [aws_security_group.bastion-elb-privatekindnet-example-com.id] + subnet_mapping { + subnet_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id + } + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_lb_listener" "bastion-privatekindnet-example-com-22" { + default_action { + target_group_arn = aws_lb_target_group.bastion-privatekindnet-ex-70mv5p.id + type = "forward" + } + load_balancer_arn = aws_lb.bastion-privatekindnet-example-com.id + port = 22 + protocol = "TCP" +} + +resource "aws_lb_target_group" "bastion-privatekindnet-ex-70mv5p" { + connection_termination = "true" + deregistration_delay = "30" + health_check { + healthy_threshold = 2 + interval = 10 + protocol = "TCP" + unhealthy_threshold = 2 + } + name = "bastion-privatekindnet-ex-70mv5p" + port = 22 + protocol = "TCP" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion-privatekindnet-ex-70mv5p" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_nat_gateway" "us-test-1a-privatekindnet-example-com" { + allocation_id = aws_eip.us-test-1a-privatekindnet-example-com.id + subnet_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.privatekindnet-example-com.id + route_table_id = aws_route_table.privatekindnet-example-com.id +} + +resource "aws_route" "route-__--0" { + destination_ipv6_cidr_block = "::/0" + gateway_id = aws_internet_gateway.privatekindnet-example-com.id + route_table_id = aws_route_table.privatekindnet-example-com.id +} + +resource "aws_route" "route-private-us-test-1a-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.us-test-1a-privatekindnet-example-com.id + route_table_id = aws_route_table.private-us-test-1a-privatekindnet-example-com.id +} + +resource "aws_route53_record" "api-privatekindnet-example-com" { + alias { + evaluate_target_health = false + name = aws_elb.api-privatekindnet-example-com.dns_name + zone_id = aws_elb.api-privatekindnet-example-com.zone_id + } + name = "api.privatekindnet.example.com" + type = "A" + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + +resource "aws_route53_record" "api-privatekindnet-example-com-AAAA" { + alias { + evaluate_target_health = false + name = aws_elb.api-privatekindnet-example-com.dns_name + zone_id = aws_elb.api-privatekindnet-example-com.zone_id + } + name = "api.privatekindnet.example.com" + type = "AAAA" + zone_id = "/hostedzone/Z1AFAKE1ZON3YO" +} + +resource "aws_route_table" "private-us-test-1a-privatekindnet-example-com" { + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "private-us-test-1a.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/kops/role" = "private-us-test-1a" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_route_table" "privatekindnet-example-com" { + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_route_table_association" "private-us-test-1a-privatekindnet-example-com" { + route_table_id = aws_route_table.private-us-test-1a-privatekindnet-example-com.id + subnet_id = aws_subnet.us-test-1a-privatekindnet-example-com.id +} + +resource "aws_route_table_association" "utility-us-test-1a-privatekindnet-example-com" { + route_table_id = aws_route_table.privatekindnet-example-com.id + subnet_id = aws_subnet.utility-us-test-1a-privatekindnet-example-com.id +} + +resource "aws_s3_object" "cluster-completed-spec" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_cluster-completed.spec_content") + key = "clusters.example.com/privatekindnet.example.com/cluster-completed.spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-events" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-events_content") + key = "clusters.example.com/privatekindnet.example.com/backups/etcd/events/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-main" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-main_content") + key = "clusters.example.com/privatekindnet.example.com/backups/etcd/main/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "kops-version-txt" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_kops-version.txt_content") + key = "clusters.example.com/privatekindnet.example.com/kops-version.txt" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-events-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-master-us-test-1a_content") + key = "clusters.example.com/privatekindnet.example.com/manifests/etcd/events-master-us-test-1a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-main-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-master-us-test-1a_content") + key = "clusters.example.com/privatekindnet.example.com/manifests/etcd/main-master-us-test-1a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content") + key = "clusters.example.com/privatekindnet.example.com/manifests/static/kube-apiserver-healthcheck.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-master-us-test-1a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-master-us-test-1a_content") + key = "clusters.example.com/privatekindnet.example.com/igconfig/control-plane/master-us-test-1a/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-nodes" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes_content") + key = "clusters.example.com/privatekindnet.example.com/igconfig/node/nodes/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-aws-cloud-controller-addons-k8s-io-k8s-1-18" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-aws-cloud-controller.addons.k8s.io-k8s-1.18_content") + key = "clusters.example.com/privatekindnet.example.com/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-aws-ebs-csi-driver-addons-k8s-io-k8s-1-17" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-aws-ebs-csi-driver.addons.k8s.io-k8s-1.17_content") + key = "clusters.example.com/privatekindnet.example.com/addons/aws-ebs-csi-driver.addons.k8s.io/k8s-1.17.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-bootstrap" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-bootstrap_content") + key = "clusters.example.com/privatekindnet.example.com/addons/bootstrap-channel.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-coredns-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content") + key = "clusters.example.com/privatekindnet.example.com/addons/coredns.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-dns-controller-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-dns-controller.addons.k8s.io-k8s-1.12_content") + key = "clusters.example.com/privatekindnet.example.com/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-kops-controller-addons-k8s-io-k8s-1-16" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-kops-controller.addons.k8s.io-k8s-1.16_content") + key = "clusters.example.com/privatekindnet.example.com/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content") + key = "clusters.example.com/privatekindnet.example.com/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-leader-migration-rbac-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-leader-migration.rbac.addons.k8s.io-k8s-1.23_content") + key = "clusters.example.com/privatekindnet.example.com/addons/leader-migration.rbac.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-limit-range-addons-k8s-io" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-limit-range.addons.k8s.io_content") + key = "clusters.example.com/privatekindnet.example.com/addons/limit-range.addons.k8s.io/v1.5.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-networking-kindnet-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-networking.kindnet-k8s-1.12_content") + key = "clusters.example.com/privatekindnet.example.com/addons/networking.kindnet/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-node-termination-handler-aws-k8s-1-11" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-node-termination-handler.aws-k8s-1.11_content") + key = "clusters.example.com/privatekindnet.example.com/addons/node-termination-handler.aws/k8s-1.11.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "privatekindnet-example-com-addons-storage-aws-addons-k8s-io-v1-15-0" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_privatekindnet.example.com-addons-storage-aws.addons.k8s.io-v1.15.0_content") + key = "clusters.example.com/privatekindnet.example.com/addons/storage-aws.addons.k8s.io/v1.15.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_security_group" "api-elb-privatekindnet-example-com" { + description = "Security group for api ELB" + name = "api-elb.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "api-elb.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "bastion-elb-privatekindnet-example-com" { + description = "Security group for bastion ELB" + name = "bastion-elb.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion-elb.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "bastion-privatekindnet-example-com" { + description = "Security group for bastion" + name = "bastion.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "bastion.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "masters-privatekindnet-example-com" { + description = "Security group for masters" + name = "masters.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "masters.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group" "nodes-privatekindnet-example-com" { + description = "Security group for nodes" + name = "nodes.privatekindnet.example.com" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "nodes.privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-22to22-bastion-elb-privatekindnet-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-0-0-0-0--0-ingress-tcp-443to443-api-elb-privatekindnet-example-com" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "from-172-20-4-0--22-ingress-tcp-22to22-bastion-elb-privatekindnet-example-com" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-api-elb-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-api-elb-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-ingress-icmp-3to4-bastion-privatekindnet-example-com" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-elb-privatekindnet-example-com-ingress-tcp-22to22-bastion-privatekindnet-example-com" { + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-ingress-icmp-3to4-bastion-elb-privatekindnet-example-com" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-ingress-tcp-22to22-masters-privatekindnet-example-com" { + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-bastion-privatekindnet-example-com-ingress-tcp-22to22-nodes-privatekindnet-example-com" { + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + source_security_group_id = aws_security_group.bastion-privatekindnet-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-ingress-all-0to0-masters-privatekindnet-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-masters-privatekindnet-example-com-ingress-all-0to0-nodes-privatekindnet-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + source_security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-egress-all-0to0-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-egress-all-0to0-__--0" { + from_port = 0 + ipv6_cidr_blocks = ["::/0"] + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-all-0to0-nodes-privatekindnet-example-com" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-tcp-1to2379-masters-privatekindnet-example-com" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-tcp-2382to4000-masters-privatekindnet-example-com" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-tcp-4003to65535-masters-privatekindnet-example-com" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "from-nodes-privatekindnet-example-com-ingress-udp-1to65535-masters-privatekindnet-example-com" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.nodes-privatekindnet-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "https-elb-to-master" { + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-cp-to-elb" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + source_security_group_id = aws_security_group.masters-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-elb-to-cp" { + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.masters-privatekindnet-example-com.id + source_security_group_id = aws_security_group.api-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_security_group_rule" "icmp-pmtu-ssh-nlb-172-20-4-0--22" { + cidr_blocks = ["172.20.4.0/22"] + from_port = 3 + protocol = "icmp" + security_group_id = aws_security_group.bastion-elb-privatekindnet-example-com.id + to_port = 4 + type = "ingress" +} + +resource "aws_sqs_queue" "privatekindnet-example-com-nth" { + message_retention_seconds = 300 + name = "privatekindnet-example-com-nth" + policy = file("${path.module}/data/aws_sqs_queue_privatekindnet-example-com-nth_policy") + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet-example-com-nth" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_subnet" "us-test-1a-privatekindnet-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "us-test-1a.privatekindnet.example.com" + "SubnetType" = "Private" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/role/internal-elb" = "1" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_subnet" "utility-us-test-1a-privatekindnet-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.4.0/22" + enable_resource_name_dns_a_record_on_launch = true + private_dns_hostname_type_on_launch = "resource-name" + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "utility-us-test-1a.privatekindnet.example.com" + "SubnetType" = "Utility" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +resource "aws_vpc" "privatekindnet-example-com" { + assign_generated_ipv6_cidr_block = true + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options" "privatekindnet-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "privatekindnet.example.com" + "Name" = "privatekindnet.example.com" + "kubernetes.io/cluster/privatekindnet.example.com" = "owned" + } +} + +resource "aws_vpc_dhcp_options_association" "privatekindnet-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.privatekindnet-example-com.id + vpc_id = aws_vpc.privatekindnet-example-com.id +} + +terraform { + required_version = ">= 0.15.0" + required_providers { + aws = { + "configuration_aliases" = [aws.files] + "source" = "hashicorp/aws" + "version" = ">= 5.0.0" + } + } +} diff --git a/upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.12.yaml.template new file mode 100644 index 0000000000000..8e5488f0d6d96 --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.kindnet/k8s-1.12.yaml.template @@ -0,0 +1,2298 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - patch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - apiGroups: + - "" + resources: + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - watch +{{- if or .Networking.Kindnet.AdminNetworkPolicies .Networking.Kindnet.BaselineAdminNetworkPolicies}} + - apiGroups: + - "policy.networking.k8s.io" + resources: + {{- if .AdminNetworkPolicies }} + - adminnetworkpolicies + {{- end }} + {{- if .BaselineAdminNetworkPolicies }} + - baselineadminnetworkpolicies + {{- end }} + verbs: + - list + - watch +{{- end }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: +- kind: ServiceAccount + name: kindnet + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kindnet + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kindnet + namespace: kube-system + labels: + tier: node + app: kindnet + k8s-app: kindnet +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + labels: + tier: node + app: kindnet + k8s-app: kindnet + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: kindnet + initContainers: + - name: install-cni-bin + image: "ghcr.io/aojea/kindnetd:{{ .Networking.Kindnet.Version }}" + command: ['sh', '-c', 'cat /opt/cni/bin/cni-kindnet > /cni/cni-kindnet ; chmod +x /cni/cni-kindnet'] + volumeMounts: + - name: cni-bin + mountPath: /cni + containers: + - name: kindnet-cni + image: "ghcr.io/aojea/kindnetd:{{ .Networking.Kindnet.Version }}" + args: + - /bin/kindnetd + - --hostname-override=$(NODE_NAME) + - --v=2 + {{- if .Networking.Kindnet.NetworkPolicies }} + - --network-policy={{ .NetworkPolicies }} + {{- end }} + {{- if .Networking.Kindnet.AdminNetworkPolicies }} + - --admin-network-policy={{ .AdminNetworkPolicies }} + {{- end }} + {{- if .Networking.Kindnet.BaselineAdminNetworkPolicies }} + - --admin-network-policy={{ .BaselineAdminNetworkPolicies }} + {{- end }} + {{- if .Networking.Kindnet.DNSCaching }} + - --dns-caching={{ .DNSCaching }} + {{- end }} + {{- if .Networking.Kindnet.NAT64 }} + - --nat64={{ .NAT64 }} + {{- end }} + {{- if .Networking.Kindnet.FastPathThreshold }} + - --fastpath-threshold={{ .FastPathThreshold }} + {{- end }} + {{- if .Networking.Kindnet.Masquerade.Enabled }} + - --masquerading={{ .Networking.Kindnet.Masquerade.Enabled }} + {{- if .Networking.Kindnet.Masquerade.NonMasqueradeCIDRs }} + - --no-masquerade-cidr={{ range $index, $element := .Networking.Kindnet.Masquerade.NonMasqueradeCIDRs }}{{if $index}},{{end}}{{$element}}{{end}} + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: cni-cfg + mountPath: /etc/cni/net.d + - name: var-lib-kindnet + mountPath: /var/lib/cni-kindnet + resources: + requests: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_RAW", "NET_ADMIN"] + volumes: + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + - name: var-lib-kindnet + hostPath: + path: /var/lib/cni-kindnet + type: DirectoryOrCreate +{{- if .Networking.Kindnet.AdminNetworkPolicies }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 + policy.networking.k8s.io/bundle-version: v0.1.1 + policy.networking.k8s.io/channel: experimental + creationTimestamp: null + name: adminnetworkpolicies.policy.networking.k8s.io +spec: + group: policy.networking.k8s.io + names: + kind: AdminNetworkPolicy + listKind: AdminNetworkPolicyList + plural: adminnetworkpolicies + shortNames: + - anp + singular: adminnetworkpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.priority + name: Priority + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + AdminNetworkPolicy is a cluster level resource that is part of the + AdminNetworkPolicy API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of AdminNetworkPolicy. + properties: + egress: + description: |- + Egress is the list of Egress rules to be applied to the selected pods. + A total of 100 rules will be allowed in each ANP instance. + The relative precedence of egress rules within a single ANP object (all of + which share the priority) will be determined by the order in which the rule + is written. Thus, a rule that appears at the top of the egress rules + would take the highest precedence. + ANPs with no egress rules do not affect egress traffic. + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressRule describes an action to take on a particular + set of traffic originating from pods selected by a AdminNetworkPolicy's + Subject field. + + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) + Deny: denies the selected traffic + Pass: instructs the selected traffic to skip any remaining ANP rules, and + then pass execution to any NetworkPolicies that select the pod. + If the pod is not selected by any NetworkPolicies then execution + is passed to any BaselineAdminNetworkPolicies that select the pod. + + Support: Core + enum: + - Allow + - Deny + - Pass + type: string + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + AdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of destination ports for the outgoing egress traffic. + If Ports is not set then the rule does not filter traffic via port. + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + to: + description: |- + To is the List of destinations whose traffic this rule applies to. + If any AdminNetworkPolicyEgressPeer matches the destination of outgoing + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + AdminNetworkPolicyEgressPeer defines a peer to allow traffic to. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + domainNames: + description: |- + DomainNames provides a way to specify domain names as peers. + + DomainNames is only supported for ALLOW rules. In order to control + access, DomainNames ALLOW rules should be used with a lower priority + egress deny -- this allows the admin to maintain an explicit "allowlist" + of reachable domains. + + DomainNames can have up to 25 domain names specified in one rule. + + Support: Extended + + + items: + description: |- + DomainName describes one or more domain names to be used as a peer. + + DomainName can be an exact match, or use the wildcard specifier '*' to match + one or more labels. + + '*', the wildcard specifier, matches one or more entire labels. It does not + support partial matches. '*' may only be specified as a prefix. + + Examples: + - `kubernetes.io` matches only `kubernetes.io`. + It does not match "www.kubernetes.io", "blog.kubernetes.io", + "my-kubernetes.io", or "wikipedia.org". + - `blog.kubernetes.io` matches only "blog.kubernetes.io". + It does not match "www.kubernetes.io" or "kubernetes.io". + - `*.kubernetes.io` matches subdomains of kubernetes.io. + "www.kubernetes.io", "blog.kubernetes.io", and + "latest.blog.kubernetes.io" match, however "kubernetes.io", and + "wikipedia.org" do not. + pattern: ^(\*\.)?([a-zA-z0-9]([-a-zA-Z0-9_]*[a-zA-Z0-9])?\.)+[a-zA-z0-9]([-a-zA-Z0-9_]*[a-zA-Z0-9])?\.?$ + type: string + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + Networks can have upto 25 CIDRs specified. + + Support: Extended + + + items: + description: CIDR is an IP address range in CIDR notation + (for example, "10.0.0.0/8" or "fd00::/8"). + maxLength: 43 + type: string + x-kubernetes-validations: + - message: Invalid CIDR format provided + rule: isCIDR(self) + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster (based on the node's labels). It selects + the nodeIPs as the peer type by matching on the IPs + present in the node.Status.Addresses field of the node. + This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + Support: Extended + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - to + type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' + maxItems: 100 + type: array + ingress: + description: |- + Ingress is the list of Ingress rules to be applied to the selected pods. + A total of 100 rules will be allowed in each ANP instance. + The relative precedence of ingress rules within a single ANP object (all of + which share the priority) will be determined by the order in which the rule + is written. Thus, a rule that appears at the top of the ingress rules + would take the highest precedence. + ANPs with no ingress rules do not affect ingress traffic. + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressRule describes an action to take on a particular + set of traffic destined for pods selected by an AdminNetworkPolicy's + Subject field. + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic (even if it would otherwise have been denied by NetworkPolicy) + Deny: denies the selected traffic + Pass: instructs the selected traffic to skip any remaining ANP rules, and + then pass execution to any NetworkPolicies that select the pod. + If the pod is not selected by any NetworkPolicies then execution + is passed to any BaselineAdminNetworkPolicies that select the pod. + + Support: Core + enum: + - Allow + - Deny + - Pass + type: string + from: + description: |- + From is the list of sources whose traffic this rule applies to. + If any AdminNetworkPolicyIngressPeer matches the source of incoming + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + AdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of ports which should be matched on + the pods selected for this policy i.e the subject of the policy. + So it matches on the destination port for the ingress traffic. + If Ports is not set then the rule does not filter traffic via port. + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - from + type: object + maxItems: 100 + type: array + priority: + description: |- + Priority is a value from 0 to 1000. Policies with lower priority values have + higher precedence, and are checked before policies with higher priority values. + All AdminNetworkPolicy rules have higher precedence than NetworkPolicy or + BaselineAdminNetworkPolicy rules + Every AdminNetworkPolicy should have a unique priority value; if two (or more) + policies with the same priority could both match a connection, then the + implementation can apply any of the matching policies to the connection, and + there is no way for the user to reliably determine which one it will choose. + + Support: Core + format: int32 + maximum: 1000 + minimum: 0 + type: integer + subject: + description: |- + Subject defines the pods to which this AdminNetworkPolicy applies. + Note that host-networked pods are not included in subject selection. + + Support: Core + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: Namespaces is used to select pods via namespace selectors. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: Pods is used to select pods via namespace AND pod + selectors. + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + required: + - priority + - subject + type: object + status: + description: Status is the status to be reported by the implementation. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +{{- end }} +{{- if .Networking.Kindnet.BaselineAdminNetworkPolicies }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/network-policy-api/pull/30 + policy.networking.k8s.io/bundle-version: v0.1.1 + policy.networking.k8s.io/channel: experimental + creationTimestamp: null + name: baselineadminnetworkpolicies.policy.networking.k8s.io +spec: + group: policy.networking.k8s.io + names: + kind: BaselineAdminNetworkPolicy + listKind: BaselineAdminNetworkPolicyList + plural: baselineadminnetworkpolicies + shortNames: + - banp + singular: baselineadminnetworkpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + BaselineAdminNetworkPolicy is a cluster level resource that is part of the + AdminNetworkPolicy API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of BaselineAdminNetworkPolicy. + properties: + egress: + description: |- + Egress is the list of Egress rules to be applied to the selected pods if + they are not matched by any AdminNetworkPolicy or NetworkPolicy rules. + A total of 100 Egress rules will be allowed in each BANP instance. + The relative precedence of egress rules within a single BANP object + will be determined by the order in which the rule is written. + Thus, a rule that appears at the top of the egress rules + would take the highest precedence. + BANPs with no egress rules do not affect egress traffic. + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyEgressRule describes an action to take on a particular + set of traffic originating from pods selected by a BaselineAdminNetworkPolicy's + Subject field. + + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic + Deny: denies the selected traffic + + Support: Core + enum: + - Allow + - Deny + type: string + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + BaselineAdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of destination ports for the outgoing egress traffic. + If Ports is not set then the rule does not filter traffic via port. + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + to: + description: |- + To is the list of destinations whose traffic this rule applies to. + If any BaselineAdminNetworkPolicyEgressPeer matches the destination of outgoing + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyEgressPeer defines a peer to allow traffic to. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + networks: + description: |- + Networks defines a way to select peers via CIDR blocks. + This is intended for representing entities that live outside the cluster, + which can't be selected by pods, namespaces and nodes peers, but note + that cluster-internal traffic will be checked against the rule as + well. So if you Allow or Deny traffic to `"0.0.0.0/0"`, that will allow + or deny all IPv4 pod-to-pod traffic as well. If you don't want that, + add a rule that Passes all pod traffic before the Networks rule. + + Each item in Networks should be provided in the CIDR format and should be + IPv4 or IPv6, for example "10.0.0.0/8" or "fd00::/8". + + Networks can have upto 25 CIDRs specified. + + Support: Extended + + + items: + description: CIDR is an IP address range in CIDR notation + (for example, "10.0.0.0/8" or "fd00::/8"). + maxLength: 43 + type: string + x-kubernetes-validations: + - message: Invalid CIDR format provided + rule: isCIDR(self) + maxItems: 25 + minItems: 1 + type: array + x-kubernetes-list-type: set + nodes: + description: |- + Nodes defines a way to select a set of nodes in + the cluster (based on the node's labels). It selects + the nodeIPs as the peer type by matching on the IPs + present in the node.Status.Addresses field of the node. + This field follows standard label selector + semantics; if present but empty, it selects all Nodes. + + Support: Extended + + + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - to + type: object + x-kubernetes-validations: + - message: networks/nodes peer cannot be set with namedPorts since + there are no namedPorts for networks/nodes + rule: '!(self.to.exists(peer, has(peer.networks) || has(peer.nodes)) + && has(self.ports) && self.ports.exists(port, has(port.namedPort)))' + maxItems: 100 + type: array + ingress: + description: |- + Ingress is the list of Ingress rules to be applied to the selected pods + if they are not matched by any AdminNetworkPolicy or NetworkPolicy rules. + A total of 100 Ingress rules will be allowed in each BANP instance. + The relative precedence of ingress rules within a single BANP object + will be determined by the order in which the rule is written. + Thus, a rule that appears at the top of the ingress rules + would take the highest precedence. + BANPs with no ingress rules do not affect ingress traffic. + + Support: Core + items: + description: |- + BaselineAdminNetworkPolicyIngressRule describes an action to take on a particular + set of traffic destined for pods selected by a BaselineAdminNetworkPolicy's + Subject field. + properties: + action: + description: |- + Action specifies the effect this rule will have on matching traffic. + Currently the following actions are supported: + Allow: allows the selected traffic + Deny: denies the selected traffic + + Support: Core + enum: + - Allow + - Deny + type: string + from: + description: |- + From is the list of sources whose traffic this rule applies to. + If any AdminNetworkPolicyIngressPeer matches the source of incoming + traffic then the specified action is applied. + This field must be defined and contain at least one item. + + Support: Core + items: + description: |- + AdminNetworkPolicyIngressPeer defines an in-cluster peer to allow traffic from. + Exactly one of the selector pointers must be set for a given peer. If a + consumer observes none of its fields are set, they must assume an unknown + option has been specified and fail closed. + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: |- + Namespaces defines a way to select all pods within a set of Namespaces. + Note that host-networked pods are not included in this type of peer. + + Support: Core + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: |- + Pods defines a way to select a set of pods in + a set of namespaces. Note that host-networked pods + are not included in this type of peer. + + Support: Core + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + name: + description: |- + Name is an identifier for this rule, that may be no more than 100 characters + in length. This field should be used by the implementation to help + improve observability, readability and error-reporting for any applied + BaselineAdminNetworkPolicies. + + Support: Core + maxLength: 100 + type: string + ports: + description: |- + Ports allows for matching traffic based on port and protocols. + This field is a list of ports which should be matched on + the pods selected for this policy i.e the subject of the policy. + So it matches on the destination port for the ingress traffic. + If Ports is not set then the rule does not filter traffic via port. + + Support: Core + items: + description: |- + AdminNetworkPolicyPort describes how to select network ports on pod(s). + Exactly one field must be set. + maxProperties: 1 + minProperties: 1 + properties: + namedPort: + description: |- + NamedPort selects a port on a pod(s) based on name. + + Support: Extended + + + type: string + portNumber: + description: |- + Port selects a port on a pod(s) based on number. + + Support: Core + properties: + port: + description: |- + Number defines a network port value. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + required: + - port + - protocol + type: object + portRange: + description: |- + PortRange selects a port range on a pod(s) based on provided start and end + values. + + Support: Core + properties: + end: + description: |- + End defines a network port that is the end of a port range, the End value + must be greater than Start. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + protocol: + default: TCP + description: |- + Protocol is the network protocol (TCP, UDP, or SCTP) which traffic must + match. If not specified, this field defaults to TCP. + + Support: Core + type: string + start: + description: |- + Start defines a network port that is the start of a port range, the Start + value must be less than End. + + Support: Core + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - end + - start + type: object + type: object + maxItems: 100 + minItems: 1 + type: array + required: + - action + - from + type: object + maxItems: 100 + type: array + subject: + description: |- + Subject defines the pods to which this BaselineAdminNetworkPolicy applies. + Note that host-networked pods are not included in subject selection. + + Support: Core + maxProperties: 1 + minProperties: 1 + properties: + namespaces: + description: Namespaces is used to select pods via namespace selectors. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + pods: + description: Pods is used to select pods via namespace AND pod + selectors. + properties: + namespaceSelector: + description: |- + NamespaceSelector follows standard label selector semantics; if empty, + it selects all Namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + podSelector: + description: |- + PodSelector is used to explicitly select pods within a namespace; if empty, + it selects all Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - namespaceSelector + - podSelector + type: object + type: object + required: + - subject + type: object + status: + description: Status is the status to be reported by the implementation. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + required: + - conditions + type: object + required: + - metadata + - spec + type: object + x-kubernetes-validations: + - message: Only one baseline admin network policy with metadata.name="default" + can be created in the cluster + rule: self.metadata.name == 'default' + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null +{{- end }} \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go index 0d758aced79e5..ee009f47af066 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/bootstrapchannelbuilder.go @@ -1070,6 +1070,23 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.CloudupModelBuilderContext) } } + if b.Cluster.Spec.Networking.Kindnet != nil { + key := "networking.kindnet" + + { + id := "k8s-1.12" + location := key + "/" + id + ".yaml" + + addons.Add(&channelsapi.AddonSpec{ + Name: fi.PtrTo(key), + Selector: networkingSelector(), + Manifest: fi.PtrTo(location), + Id: id, + NeedsRollingUpdate: channelsapi.NeedsRollingUpdateAll, + }) + } + } + err := addCiliumAddon(b, addons) if err != nil { return nil, nil, fmt.Errorf("failed to add cilium addon: %w", err) diff --git a/upup/pkg/fi/cloudup/new_cluster.go b/upup/pkg/fi/cloudup/new_cluster.go index e028b63f82cee..38b1f2a0f83f7 100644 --- a/upup/pkg/fi/cloudup/new_cluster.go +++ b/upup/pkg/fi/cloudup/new_cluster.go @@ -1262,6 +1262,8 @@ func setupNetworking(opt *NewClusterOptions, cluster *api.Cluster) error { cluster.Spec.Networking.Cilium.IPAM = "eni" case "gcp", "gce": cluster.Spec.Networking.GCP = &api.GCPNetworkingSpec{} + case "kindnet": + cluster.Spec.Networking.Kindnet = &api.KindnetNetworkingSpec{} default: return fmt.Errorf("unknown networking mode %q", opt.Networking) } diff --git a/upup/pkg/fi/cloudup/new_cluster_test.go b/upup/pkg/fi/cloudup/new_cluster_test.go index 20c3351ab0ae1..c2d1bdd16b41c 100644 --- a/upup/pkg/fi/cloudup/new_cluster_test.go +++ b/upup/pkg/fi/cloudup/new_cluster_test.go @@ -313,6 +313,18 @@ func TestSetupNetworking(t *testing.T) { }, }, }, + { + options: NewClusterOptions{ + Networking: "kindnet", + }, + expected: api.Cluster{ + Spec: api.ClusterSpec{ + Networking: api.NetworkingSpec{ + Kindnet: &api.KindnetNetworkingSpec{}, + }, + }, + }, + }, } for _, test := range tests { diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index c56608600e28f..2bc88ac846efc 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -318,6 +318,7 @@ func (c *populateClusterSpec) run(ctx context.Context, clientset simple.Clientse codeModels = append(codeModels, &components.CloudConfigurationOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.CalicoOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.CiliumOptionsBuilder{Context: optionsContext}) + codeModels = append(codeModels, &components.KindnetOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.OpenStackOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.DiscoveryOptionsBuilder{OptionsContext: optionsContext}) codeModels = append(codeModels, &components.ClusterAutoscalerOptionsBuilder{OptionsContext: optionsContext}) diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 5590166776e69..9bde31d6affb8 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -322,6 +322,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &networking.CommonBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CiliumBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &networking.KindnetBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.AmazonVPCRoutedENIBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.KuberouterBuilder{NodeupModelContext: modelContext})