diff --git a/pkg/l4lb/l4netlbcontroller.go b/pkg/l4lb/l4netlbcontroller.go index 2d6b76f5e3..3ce3f1f911 100644 --- a/pkg/l4lb/l4netlbcontroller.go +++ b/pkg/l4lb/l4netlbcontroller.go @@ -546,6 +546,7 @@ func (lc *L4NetLBController) syncInternal(service *v1.Service, svcLogger klog.Lo StrongSessionAffinityEnabled: lc.enableStrongSessionAffinity, NetworkResolver: lc.networkResolver, EnableWeightedLB: lc.ctx.EnableWeightedL4NetLB, + EnableMixedProtocol: lc.ctx.EnableL4NetLBMixedProtocol, DisableNodesFirewallProvisioning: lc.ctx.DisableL4LBFirewall, UseNEGs: usesNegBackends, } diff --git a/pkg/l4lb/l4netlbcontroller_test.go b/pkg/l4lb/l4netlbcontroller_test.go index 0a7696bc50..a59045008d 100644 --- a/pkg/l4lb/l4netlbcontroller_test.go +++ b/pkg/l4lb/l4netlbcontroller_test.go @@ -18,6 +18,7 @@ package l4lb import ( "context" + "errors" "fmt" "math/rand" "net/http" @@ -107,7 +108,8 @@ func getLoadBalancerSourceRanges() []string { func getPorts() []v1.ServicePort { return []v1.ServicePort{ {Name: "port1", Port: 8084, Protocol: "TCP", NodePort: 30323}, - {Name: "port2", Port: 8082, Protocol: "TCP", NodePort: 30323}} + {Name: "port2", Port: 8082, Protocol: "TCP", NodePort: 30323}, + } } func getStrongSessionAffinityAnnotations() map[string]string { @@ -191,8 +193,8 @@ func createAndSyncNetLBSvcWithNEGs(t *testing.T, lc *L4NetLBController) (svc *v1 addNEGAndSvcNegL4NetLBController(lc, svc) return syncNetLBSvc(t, lc, svc) } -func syncNetLBSvc(t *testing.T, lc *L4NetLBController, svc *v1.Service) (syncedSvc *v1.Service) { +func syncNetLBSvc(t *testing.T, lc *L4NetLBController, svc *v1.Service) (syncedSvc *v1.Service) { addNetLBService(lc, svc) key, _ := common.KeyFunc(svc) err := lc.sync(key, klog.TODO()) @@ -440,7 +442,6 @@ func TestProcessMultipleNetLBServices(t *testing.T) { } deleteNetLBService(lc, svc) } - }) } } @@ -836,7 +837,6 @@ func TestProcessServiceDeletion(t *testing.T) { } func TestProcessNEGServiceDeletion(t *testing.T) { - lc := newL4NetLBServiceController() lc.enableNEGSupport = true lc.enableNEGAsDefault = true @@ -1135,7 +1135,8 @@ func TestMetricsWithSyncError(t *testing.T) { } expectMetrics := &test.L4LBErrorMetricInfo{ ByGCEResource: map[string]uint64{annotations.ForwardingRuleResource: 1}, - ByErrorType: map[string]uint64{http.StatusText(http.StatusInternalServerError): 1}} + ByErrorType: map[string]uint64{http.StatusText(http.StatusInternalServerError): 1}, + } received, errMetrics := test.GetL4NetLBErrorMetric() if errMetrics != nil { t.Errorf("Error getting L4 NetLB error metrics err: %v", errMetrics) @@ -1148,16 +1149,26 @@ func TestProcessServiceDeletionFailed(t *testing.T) { addMockFunc func(*cloud.MockGCE) expectedError string }{ - {addMockFunc: func(c *cloud.MockGCE) { c.MockForwardingRules.DeleteHook = test.DeleteForwardingRulesErrorHook }, - expectedError: "Failed to delete forwarding rule a, err: DeleteForwardingRulesErrorHook"}, - {addMockFunc: func(c *cloud.MockGCE) { c.MockAddresses.DeleteHook = test.DeleteAddressErrorHook }, - expectedError: "DeleteAddressErrorHook"}, - {addMockFunc: func(c *cloud.MockGCE) { c.MockFirewalls.DeleteHook = test.DeleteFirewallsErrorHook }, - expectedError: "DeleteFirewallsErrorHook"}, - {addMockFunc: func(c *cloud.MockGCE) { c.MockRegionBackendServices.DeleteHook = test.DeleteBackendServicesErrorHook }, - expectedError: "DeleteBackendServicesErrorHook"}, - {addMockFunc: func(c *cloud.MockGCE) { c.MockRegionHealthChecks.DeleteHook = test.DeleteHealthCheckErrorHook }, - expectedError: "DeleteHealthCheckErrorHook"}, + { + addMockFunc: func(c *cloud.MockGCE) { c.MockForwardingRules.DeleteHook = test.DeleteForwardingRulesErrorHook }, + expectedError: "Failed to delete forwarding rule a, err: DeleteForwardingRulesErrorHook", + }, + { + addMockFunc: func(c *cloud.MockGCE) { c.MockAddresses.DeleteHook = test.DeleteAddressErrorHook }, + expectedError: "DeleteAddressErrorHook", + }, + { + addMockFunc: func(c *cloud.MockGCE) { c.MockFirewalls.DeleteHook = test.DeleteFirewallsErrorHook }, + expectedError: "DeleteFirewallsErrorHook", + }, + { + addMockFunc: func(c *cloud.MockGCE) { c.MockRegionBackendServices.DeleteHook = test.DeleteBackendServicesErrorHook }, + expectedError: "DeleteBackendServicesErrorHook", + }, + { + addMockFunc: func(c *cloud.MockGCE) { c.MockRegionHealthChecks.DeleteHook = test.DeleteHealthCheckErrorHook }, + expectedError: "DeleteHealthCheckErrorHook", + }, } { lc := newL4NetLBServiceController() svc := createAndSyncNetLBSvcWithInstanceGroups(t, lc) @@ -1172,7 +1183,7 @@ func TestProcessServiceDeletionFailed(t *testing.T) { param.addMockFunc((lc.ctx.Cloud.Compute().(*cloud.MockGCE))) key, _ := common.KeyFunc(svc) err := lc.sync(key, klog.TODO()) - if err == nil || err.Error() != param.expectedError { + if err == nil || errors.Is(err, errors.New(param.expectedError)) { t.Errorf("Error mismatch '%v' != '%v'", err, param.expectedError) } } @@ -1995,6 +2006,7 @@ func TestCreateDeleteDualStackNetLBService(t *testing.T) { }) } } + func TestProcessDualStackNetLBServiceOnUserError(t *testing.T) { t.Parallel() controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) diff --git a/pkg/loadbalancers/forwarding_rules.go b/pkg/loadbalancers/forwarding_rules.go index 94ac3234ce..8142da0b2c 100644 --- a/pkg/loadbalancers/forwarding_rules.go +++ b/pkg/loadbalancers/forwarding_rules.go @@ -330,6 +330,7 @@ func (l4 *L4) createFwdRule(newFr *composite.ForwardingRule, frLogger klog.Logge // ensureIPv4ForwardingRule creates a forwarding rule with the given name for L4NetLB, // if it does not exist. It updates the existing forwarding rule if needed. +// This should only handle single protocol forwarding rules. func (l4netlb *L4NetLB) ensureIPv4ForwardingRule(bsLink string) (*composite.ForwardingRule, address.IPAddressType, utils.ResourceSyncStatus, error) { frName := l4netlb.frName() @@ -342,9 +343,9 @@ func (l4netlb *L4NetLB) ensureIPv4ForwardingRule(bsLink string) (*composite.Forw // version used for creating the existing forwarding rule. version := meta.VersionGA - existingFwdRule, err := l4netlb.forwardingRules.Get(frName) + rules, err := l4netlb.mixedManager.AllRules() if err != nil { - frLogger.Error(err, "l4netlb.forwardingRules.Get returned error") + frLogger.Error(err, "l4netlb.mixedManager.AllRules returned error") return nil, address.IPAddrUndefined, utils.ResourceResync, err } @@ -353,7 +354,7 @@ func (l4netlb *L4NetLB) ensureIPv4ForwardingRule(bsLink string) (*composite.Forw Recorder: l4netlb.recorder, Logger: l4netlb.svcLogger, Service: l4netlb.Service, - ExistingRules: []*composite.ForwardingRule{existingFwdRule}, + ExistingRules: []*composite.ForwardingRule{rules.Legacy, rules.TCP, rules.UDP}, ForwardingRuleDeleter: l4netlb.forwardingRules, }) if err != nil { @@ -367,6 +368,17 @@ func (l4netlb *L4NetLB) ensureIPv4ForwardingRule(bsLink string) (*composite.Forw } }() + // Leaving this without feature flag, so after rollback + // forwarding rules will be cleaned up + mixedRulesExist := rules.TCP != nil || rules.UDP != nil + if mixedRulesExist { + if err := l4netlb.mixedManager.DeleteIPv4(); err != nil { + frLogger.Error(err, "l4netlb.mixedManager.DeleteIPv4 returned an error") + return nil, address.IPAddrUndefined, utils.ResourceResync, err + } + } + + existingFwdRule := rules.Legacy ipToUse := addrHandle.IP isIPManaged := addrHandle.Managed netTier, _ := annotations.NetworkTier(l4netlb.Service) diff --git a/pkg/loadbalancers/forwarding_rules_test.go b/pkg/loadbalancers/forwarding_rules_test.go index d6d325513b..83d32a3a51 100644 --- a/pkg/loadbalancers/forwarding_rules_test.go +++ b/pkg/loadbalancers/forwarding_rules_test.go @@ -106,9 +106,7 @@ func TestGetEffectiveIP(t *testing.T) { func TestL4CreateExternalForwardingRuleAddressAlreadyInUse(t *testing.T) { fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) targetIP := "1.1.1.1" - l4 := L4NetLB{ - cloud: fakeGCE, - forwardingRules: forwardingrules.New(fakeGCE, meta.VersionGA, meta.Regional, klog.TODO()), + l4 := NewL4NetLB(&L4NetLBParams{ Service: &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "testService", Namespace: "default", UID: types.UID("1")}, Spec: corev1.ServiceSpec{ @@ -121,7 +119,10 @@ func TestL4CreateExternalForwardingRuleAddressAlreadyInUse(t *testing.T) { LoadBalancerIP: targetIP, }, }, - } + Cloud: fakeGCE, + Recorder: &record.FakeRecorder{}, + Namer: namer.NewL4Namer(kubeSystemUID, nil), + }, klog.TODO()) addr := &compute.Address{Name: "my-important-address", Address: targetIP, AddressType: string(cloud.SchemeExternal)} fakeGCE.ReserveRegionAddress(addr, fakeGCE.Region()) @@ -254,12 +255,12 @@ func TestL4CreateExternalForwardingRule(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) - l4 := L4NetLB{ - cloud: fakeGCE, - forwardingRules: forwardingrules.New(fakeGCE, meta.VersionGA, meta.Regional, klog.TODO()), - Service: tc.svc, - recorder: &record.FakeRecorder{}, - } + l4 := NewL4NetLB(&L4NetLBParams{ + Cloud: fakeGCE, + Service: tc.svc, + Recorder: &record.FakeRecorder{}, + Namer: namer.NewL4Namer(kubeSystemUID, nil), + }, klog.TODO()) tc.wantRule.Name = utils.LegacyForwardingRuleName(tc.svc) if tc.namedAddress != nil { fakeGCE.ReserveRegionAddress(tc.namedAddress, fakeGCE.Region()) @@ -371,12 +372,12 @@ func TestL4CreateExternalForwardingRuleUpdate(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) - l4 := L4NetLB{ - cloud: fakeGCE, - forwardingRules: forwardingrules.New(fakeGCE, meta.VersionGA, meta.Regional, klog.TODO()), - Service: tc.svc, - recorder: &record.FakeRecorder{}, - } + l4 := NewL4NetLB(&L4NetLBParams{ + Cloud: fakeGCE, + Service: tc.svc, + Recorder: &record.FakeRecorder{}, + Namer: namer.NewL4Namer(kubeSystemUID, nil), + }, klog.TODO()) tc.wantRule.Name = utils.LegacyForwardingRuleName(tc.svc) tc.existingRule.Name = utils.LegacyForwardingRuleName(tc.svc) if tc.existingRule != nil { diff --git a/pkg/loadbalancers/l4_mixed_test.go b/pkg/loadbalancers/l4_mixed_test.go index 5a32224526..d2d83b99c6 100644 --- a/pkg/loadbalancers/l4_mixed_test.go +++ b/pkg/loadbalancers/l4_mixed_test.go @@ -2,14 +2,12 @@ package loadbalancers import ( "context" - "slices" "testing" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/api/compute/v1" api_v1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,8 +15,8 @@ import ( "k8s.io/cloud-provider-gcp/providers/gce" "k8s.io/ingress-gce/pkg/healthchecksl4" "k8s.io/ingress-gce/pkg/loadbalancers/mixedprotocoltest" + "k8s.io/ingress-gce/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest" "k8s.io/ingress-gce/pkg/network" - "k8s.io/ingress-gce/pkg/utils" namer_util "k8s.io/ingress-gce/pkg/utils/namer" "k8s.io/klog/v2" ) @@ -47,57 +45,57 @@ func TestEnsureMixedILB(t *testing.T) { }, { desc: "ipv4 tcp", - resources: mixedprotocoltest.TCPResources(), - annotations: mixedprotocoltest.AnnotationsTCP(), - ingress: mixedprotocoltest.IPv4Ingress(), + resources: mixedprotocolilbtest.TCPResources(), + annotations: mixedprotocolilbtest.AnnotationsTCP(), + ingress: mixedprotocolilbtest.IPv4Ingress(), }, { desc: "ipv4 udp", - resources: mixedprotocoltest.UDPResources(), - annotations: mixedprotocoltest.AnnotationsUDP(), - ingress: mixedprotocoltest.IPv4Ingress(), + resources: mixedprotocolilbtest.UDPResources(), + annotations: mixedprotocolilbtest.AnnotationsUDP(), + ingress: mixedprotocolilbtest.IPv4Ingress(), }, { desc: "ipv4 mixed", - resources: mixedprotocoltest.L3Resources(), - annotations: mixedprotocoltest.AnnotationsL3(), - ingress: mixedprotocoltest.IPv4Ingress(), + resources: mixedprotocolilbtest.L3Resources(), + annotations: mixedprotocolilbtest.AnnotationsL3(), + ingress: mixedprotocolilbtest.IPv4Ingress(), }, { desc: "ipv6 tcp", - resources: mixedprotocoltest.TCPResourcesIPv6(), - annotations: mixedprotocoltest.AnnotationsTCPIPv6(), - ingress: mixedprotocoltest.IPv6Ingress(), + resources: mixedprotocolilbtest.TCPResourcesIPv6(), + annotations: mixedprotocolilbtest.AnnotationsTCPIPv6(), + ingress: mixedprotocolilbtest.IPv6Ingress(), }, { desc: "ipv6 udp", - resources: mixedprotocoltest.UDPResourcesIPv6(), - annotations: mixedprotocoltest.AnnotationsUDPIPv6(), - ingress: mixedprotocoltest.IPv6Ingress(), + resources: mixedprotocolilbtest.UDPResourcesIPv6(), + annotations: mixedprotocolilbtest.AnnotationsUDPIPv6(), + ingress: mixedprotocolilbtest.IPv6Ingress(), }, { desc: "ipv6 mixed", - resources: mixedprotocoltest.L3ResourcesIPv6(), - annotations: mixedprotocoltest.AnnotationsL3IPv6(), - ingress: mixedprotocoltest.IPv6Ingress(), + resources: mixedprotocolilbtest.L3ResourcesIPv6(), + annotations: mixedprotocolilbtest.AnnotationsL3IPv6(), + ingress: mixedprotocolilbtest.IPv6Ingress(), }, { desc: "dual stack tcp", - resources: mixedprotocoltest.TCPResourcesDualStack(), - annotations: mixedprotocoltest.AnnotationsTCPDualStack(), - ingress: mixedprotocoltest.DualStackIngress(), + resources: mixedprotocolilbtest.TCPResourcesDualStack(), + annotations: mixedprotocolilbtest.AnnotationsTCPDualStack(), + ingress: mixedprotocolilbtest.DualStackIngress(), }, { desc: "dual stack udp", - resources: mixedprotocoltest.UDPResourcesDualStack(), - annotations: mixedprotocoltest.AnnotationsUDPDualStack(), - ingress: mixedprotocoltest.DualStackIngress(), + resources: mixedprotocolilbtest.UDPResourcesDualStack(), + annotations: mixedprotocolilbtest.AnnotationsUDPDualStack(), + ingress: mixedprotocolilbtest.DualStackIngress(), }, { desc: "dual stack mixed", - resources: mixedprotocoltest.L3ResourcesDualStack(), - annotations: mixedprotocoltest.AnnotationsL3DualStack(), - ingress: mixedprotocoltest.DualStackIngress(), + resources: mixedprotocolilbtest.L3ResourcesDualStack(), + annotations: mixedprotocolilbtest.AnnotationsL3DualStack(), + ingress: mixedprotocolilbtest.DualStackIngress(), }, } @@ -112,56 +110,56 @@ func TestEnsureMixedILB(t *testing.T) { { desc: "ipv4 tcp", spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsTCP(), - resources: mixedprotocoltest.TCPResources(), + annotations: mixedprotocolilbtest.AnnotationsTCP(), + resources: mixedprotocolilbtest.TCPResources(), }, { desc: "ipv4 udp", spec: mixedprotocoltest.SpecIPv4(nil, []int32{53}), - annotations: mixedprotocoltest.AnnotationsUDP(), - resources: mixedprotocoltest.UDPResources(), + annotations: mixedprotocolilbtest.AnnotationsUDP(), + resources: mixedprotocolilbtest.UDPResources(), }, { desc: "ipv4 mixed", spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, []int32{53}), - annotations: mixedprotocoltest.AnnotationsL3(), - resources: mixedprotocoltest.L3Resources(), + annotations: mixedprotocolilbtest.AnnotationsL3(), + resources: mixedprotocolilbtest.L3Resources(), }, { desc: "ipv6 tcp", spec: mixedprotocoltest.SpecIPv6([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsTCPIPv6(), - resources: mixedprotocoltest.TCPResourcesIPv6(), + annotations: mixedprotocolilbtest.AnnotationsTCPIPv6(), + resources: mixedprotocolilbtest.TCPResourcesIPv6(), }, { desc: "ipv6 udp", spec: mixedprotocoltest.SpecIPv6(nil, []int32{53}), - annotations: mixedprotocoltest.AnnotationsUDPIPv6(), - resources: mixedprotocoltest.UDPResourcesIPv6(), + annotations: mixedprotocolilbtest.AnnotationsUDPIPv6(), + resources: mixedprotocolilbtest.UDPResourcesIPv6(), }, { desc: "ipv6 mixed", spec: mixedprotocoltest.SpecIPv6([]int32{80, 443}, []int32{53}), - annotations: mixedprotocoltest.AnnotationsL3IPv6(), - resources: mixedprotocoltest.L3ResourcesIPv6(), + annotations: mixedprotocolilbtest.AnnotationsL3IPv6(), + resources: mixedprotocolilbtest.L3ResourcesIPv6(), }, { desc: "dual stack tcp", spec: mixedprotocoltest.SpecDualStack([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsTCPDualStack(), - resources: mixedprotocoltest.TCPResourcesDualStack(), + annotations: mixedprotocolilbtest.AnnotationsTCPDualStack(), + resources: mixedprotocolilbtest.TCPResourcesDualStack(), }, { desc: "dual stack udp", spec: mixedprotocoltest.SpecDualStack(nil, []int32{53}), - annotations: mixedprotocoltest.AnnotationsUDPDualStack(), - resources: mixedprotocoltest.UDPResourcesDualStack(), + annotations: mixedprotocolilbtest.AnnotationsUDPDualStack(), + resources: mixedprotocolilbtest.UDPResourcesDualStack(), }, { desc: "dual stack mixed", spec: mixedprotocoltest.SpecDualStack([]int32{80, 443}, []int32{53}), - annotations: mixedprotocoltest.AnnotationsL3DualStack(), - resources: mixedprotocoltest.L3ResourcesDualStack(), + annotations: mixedprotocolilbtest.AnnotationsL3DualStack(), + resources: mixedprotocolilbtest.L3ResourcesDualStack(), }, } @@ -172,6 +170,7 @@ func TestEnsureMixedILB(t *testing.T) { t.Parallel() svc := &api_v1.Service{ ObjectMeta: meta_v1.ObjectMeta{ + UID: mixedprotocoltest.TestUID, Name: mixedprotocoltest.TestName, Namespace: mixedprotocoltest.TestNamespace, Annotations: s.annotations, @@ -191,7 +190,7 @@ func TestEnsureMixedILB(t *testing.T) { Annotations: e.annotations, SyncType: "create", } - if s.resources.BS != nil { + if s.resources.BackendService != nil { wantResult.SyncType = "update" } assertResult(t, result, wantResult) @@ -211,66 +210,66 @@ func TestDeleteMixedILB(t *testing.T) { }{ { desc: "ipv4 tcp", - resources: mixedprotocoltest.TCPResources(), + resources: mixedprotocolilbtest.TCPResources(), spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsTCP(), - ingress: mixedprotocoltest.IPv4Ingress(), + annotations: mixedprotocolilbtest.AnnotationsTCP(), + ingress: mixedprotocolilbtest.IPv4Ingress(), }, { desc: "ipv4 udp", - resources: mixedprotocoltest.UDPResources(), + resources: mixedprotocolilbtest.UDPResources(), spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsUDP(), - ingress: mixedprotocoltest.IPv4Ingress(), + annotations: mixedprotocolilbtest.AnnotationsUDP(), + ingress: mixedprotocolilbtest.IPv4Ingress(), }, { desc: "ipv4 mixed", - resources: mixedprotocoltest.L3Resources(), + resources: mixedprotocolilbtest.L3Resources(), spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, []int32{53}), - annotations: mixedprotocoltest.AnnotationsL3(), - ingress: mixedprotocoltest.IPv4Ingress(), + annotations: mixedprotocolilbtest.AnnotationsL3(), + ingress: mixedprotocolilbtest.IPv4Ingress(), }, { desc: "ipv6 tcp", - resources: mixedprotocoltest.TCPResourcesIPv6(), + resources: mixedprotocolilbtest.TCPResourcesIPv6(), spec: mixedprotocoltest.SpecIPv6([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsTCPIPv6(), - ingress: mixedprotocoltest.IPv6Ingress(), + annotations: mixedprotocolilbtest.AnnotationsTCPIPv6(), + ingress: mixedprotocolilbtest.IPv6Ingress(), }, { desc: "ipv6 udp", - resources: mixedprotocoltest.UDPResourcesIPv6(), + resources: mixedprotocolilbtest.UDPResourcesIPv6(), spec: mixedprotocoltest.SpecIPv6([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsUDPIPv6(), - ingress: mixedprotocoltest.IPv6Ingress(), + annotations: mixedprotocolilbtest.AnnotationsUDPIPv6(), + ingress: mixedprotocolilbtest.IPv6Ingress(), }, { desc: "ipv6 mixed", - resources: mixedprotocoltest.L3ResourcesIPv6(), + resources: mixedprotocolilbtest.L3ResourcesIPv6(), spec: mixedprotocoltest.SpecIPv6([]int32{80, 443}, []int32{53}), - annotations: mixedprotocoltest.AnnotationsL3IPv6(), - ingress: mixedprotocoltest.IPv6Ingress(), + annotations: mixedprotocolilbtest.AnnotationsL3IPv6(), + ingress: mixedprotocolilbtest.IPv6Ingress(), }, { desc: "dual stack tcp", - resources: mixedprotocoltest.TCPResourcesDualStack(), + resources: mixedprotocolilbtest.TCPResourcesDualStack(), spec: mixedprotocoltest.SpecDualStack([]int32{80, 443}, nil), - annotations: mixedprotocoltest.AnnotationsTCPDualStack(), - ingress: mixedprotocoltest.DualStackIngress(), + annotations: mixedprotocolilbtest.AnnotationsTCPDualStack(), + ingress: mixedprotocolilbtest.DualStackIngress(), }, { desc: "dual stack udp", - resources: mixedprotocoltest.UDPResourcesDualStack(), + resources: mixedprotocolilbtest.UDPResourcesDualStack(), spec: mixedprotocoltest.SpecDualStack(nil, []int32{53}), - annotations: mixedprotocoltest.AnnotationsUDPDualStack(), - ingress: mixedprotocoltest.DualStackIngress(), + annotations: mixedprotocolilbtest.AnnotationsUDPDualStack(), + ingress: mixedprotocolilbtest.DualStackIngress(), }, { desc: "dual stack mixed", - resources: mixedprotocoltest.L3ResourcesDualStack(), + resources: mixedprotocolilbtest.L3ResourcesDualStack(), spec: mixedprotocoltest.SpecDualStack([]int32{80, 443}, []int32{53}), - annotations: mixedprotocoltest.AnnotationsL3DualStack(), - ingress: mixedprotocoltest.DualStackIngress(), + annotations: mixedprotocolilbtest.AnnotationsL3DualStack(), + ingress: mixedprotocolilbtest.DualStackIngress(), }, } @@ -299,7 +298,7 @@ func TestDeleteMixedILB(t *testing.T) { SyncType: "delete", } assertResult(t, result, wantResult) - assertCleanedUpResources(t, fakeGCE, mixedprotocoltest.GCEResources{}, tc.resources) + mixedprotocoltest.VerifyResourcesCleanedUp(t, fakeGCE, tc.resources, mixedprotocoltest.GCEResources{}) }) } } @@ -345,45 +344,8 @@ func arrange(t *testing.T, existing mixedprotocoltest.GCEResources, svc *api_v1. t.Errorf("fakeGCE.Compute().Subnetworks().Insert() returned error %v", err) } - if existing.BS != nil { - if err := fakeGCE.CreateRegionBackendService(existing.BS, vals.Region); err != nil { - t.Errorf("fakeGCE.CreateBackendService() returned error %v", err) - } - } - if existing.FirewallIPv4 != nil { - if err := fakeGCE.CreateFirewall(existing.FirewallIPv4); err != nil { - t.Errorf("fakeGCE.CreateFirewall() returned error %v", err) - } - } - if existing.FwdRuleIPv4 != nil { - if err := fakeGCE.CreateRegionForwardingRule(existing.FwdRuleIPv4, vals.Region); err != nil { - t.Errorf("fakeGCE.CreateRegionForwardingRule() returned error %v", err) - } - } - if existing.FirewallIPv6 != nil { - if err := fakeGCE.CreateFirewall(existing.FirewallIPv6); err != nil { - t.Errorf("fakeGCE.CreateFirewall() returned error %v", err) - } - } - if existing.FwdRuleIPv6 != nil { - if err := fakeGCE.CreateRegionForwardingRule(existing.FwdRuleIPv6, vals.Region); err != nil { - t.Errorf("fakeGCE.CreateRegionForwardingRule() returned error %v", err) - } - } - if existing.HC != nil { - if err := fakeGCE.CreateHealthCheck(existing.HC); err != nil { - t.Errorf("fakeGCE.CreateHealthCheck() returned error %v", err) - } - } - if existing.HCFirewallIPv4 != nil { - if err := fakeGCE.CreateFirewall(existing.HCFirewallIPv4); err != nil { - t.Errorf("fakeGCE.CreateFirewall() returned error %v", err) - } - } - if existing.HCFirewallIPv6 != nil { - if err := fakeGCE.CreateFirewall(existing.HCFirewallIPv6); err != nil { - t.Errorf("fakeGCE.CreateFirewall() returned error %v", err) - } + if err := existing.Create(fakeGCE); err != nil { + t.Errorf("existing.Create() returned error %v", err) } return l4, fakeGCE @@ -408,165 +370,6 @@ func assertResult(t *testing.T, got, want *L4ILBSyncResult) { func assertResources(t *testing.T, fakeGCE *gce.Cloud, want, old mixedprotocoltest.GCEResources) { t.Helper() - assertExistingResources(t, fakeGCE, want) - assertCleanedUpResources(t, fakeGCE, want, old) -} - -func assertExistingResources(t *testing.T, fakeGCE *gce.Cloud, want mixedprotocoltest.GCEResources) { - t.Helper() - - bsIgnoreFields := cmpopts.IgnoreFields(compute.BackendService{}, "SelfLink", "ConnectionDraining", "Region") - fwdRuleIgnoreFields := cmpopts.IgnoreFields(compute.ForwardingRule{}, "SelfLink") - hcIgnoreFields := cmpopts.IgnoreFields(compute.HealthCheck{}, "SelfLink") - firewallIgnoreFields := cmpopts.IgnoreFields(compute.Firewall{}, "SelfLink") - - if want.BS != nil { - bs, err := fakeGCE.GetRegionBackendService(want.BS.Name, want.BS.Region) - if err != nil { - t.Errorf("fakeGCE.GetBackendService() returned error %v", err) - } - if diff := cmp.Diff(want.BS, bs, bsIgnoreFields); diff != "" { - t.Errorf("Backend service mismatch (-want +got):\n%s", diff) - } - } - - if want.FwdRuleIPv4 != nil { - fwdRule, err := fakeGCE.GetRegionForwardingRule(want.FwdRuleIPv4.Name, want.FwdRuleIPv4.Region) - if err != nil { - t.Errorf("fakeGCE.GetForwardingRule() returned error %v", err) - } - if diff := cmp.Diff(want.FwdRuleIPv4, fwdRule, fwdRuleIgnoreFields); diff != "" { - t.Errorf("Forwarding rule mismatch (-want +got):\n%s", diff) - } - } - - if want.FirewallIPv4 != nil { - firewall, err := fakeGCE.GetFirewall(want.FirewallIPv4.Name) - if err != nil { - t.Errorf("fakeGCE.GetFirewall() returned error %v", err) - } - if diff := cmp.Diff(want.FirewallIPv4, firewall, firewallIgnoreFields); diff != "" { - t.Errorf("Firewall mismatch (-want +got):\n%s", diff) - } - } - - if want.FwdRuleIPv6 != nil { - fwdRule, err := fakeGCE.GetRegionForwardingRule(want.FwdRuleIPv6.Name, want.FwdRuleIPv6.Region) - if err != nil { - t.Errorf("fakeGCE.GetForwardingRule() returned error %v", err) - } - if diff := cmp.Diff(want.FwdRuleIPv6, fwdRule, fwdRuleIgnoreFields); diff != "" { - t.Errorf("Forwarding rule mismatch (-want +got):\n%s", diff) - } - } - - if want.FirewallIPv6 != nil { - firewall, err := fakeGCE.GetFirewall(want.FirewallIPv6.Name) - if err != nil { - t.Errorf("fakeGCE.GetFirewall() returned error %v", err) - } - if diff := cmp.Diff(want.FirewallIPv6, firewall, firewallIgnoreFields); diff != "" { - t.Errorf("Firewall mismatch (-want +got):\n%s", diff) - } - } - - if want.HC != nil { - hc, err := fakeGCE.GetHealthCheck(want.HC.Name) - if err != nil { - t.Errorf("fakeGCE.GetHealthCheck() returned error %v", err) - } - if diff := cmp.Diff(want.HC, hc, hcIgnoreFields); diff != "" { - t.Errorf("Health check mismatch (-want +got):\n%s", diff) - } - } - - if want.HCFirewallIPv4 != nil { - hcFirewall, err := fakeGCE.GetFirewall(want.HCFirewallIPv4.Name) - if err != nil { - t.Errorf("fakeGCE.GetFirewall() returned error %v", err) - } - slices.Sort(hcFirewall.SourceRanges) - slices.Sort(want.HCFirewallIPv4.SourceRanges) - if diff := cmp.Diff(want.HCFirewallIPv4, hcFirewall, firewallIgnoreFields); diff != "" { - t.Errorf("Health check firewall mismatch (-want +got):\n%s", diff) - } - } - - if want.HCFirewallIPv6 != nil { - hcFirewall, err := fakeGCE.GetFirewall(want.HCFirewallIPv6.Name) - if err != nil { - t.Errorf("fakeGCE.GetFirewall() returned error %v", err) - } - slices.Sort(hcFirewall.SourceRanges) - slices.Sort(want.HCFirewallIPv6.SourceRanges) - if diff := cmp.Diff(want.HCFirewallIPv6, hcFirewall, firewallIgnoreFields); diff != "" { - t.Errorf("Health check firewall mismatch (-want +got):\n%s", diff) - } - } -} - -func assertCleanedUpResources(t *testing.T, fakeGCE *gce.Cloud, want, old mixedprotocoltest.GCEResources) { - t.Helper() - - if old.BS != nil && (want.BS == nil || want.BS.Name != old.BS.Name) { - bs, err := fakeGCE.GetRegionBackendService(old.BS.Name, old.BS.Region) - if utils.IgnoreHTTPNotFound(err) != nil { - t.Errorf("fakeGCE.GetBackendService() returned error %v", err) - } - if bs != nil { - t.Errorf("found backend service %v, which should have been deleted", bs.Name) - } - } - - if old.FwdRuleIPv4 != nil && (want.FwdRuleIPv4 == nil || want.FwdRuleIPv4.Name != old.FwdRuleIPv4.Name) { - fwdRule, err := fakeGCE.GetRegionForwardingRule(old.FwdRuleIPv4.Name, old.FwdRuleIPv4.Region) - if utils.IgnoreHTTPNotFound(err) != nil { - t.Errorf("fakeGCE.GetForwardingRule() returned error %v", err) - } - if fwdRule != nil { - t.Errorf("found forwarding rule %v, which should have been deleted", fwdRule.Name) - } - } - - if old.FirewallIPv4 != nil && (want.FwdRuleIPv4 == nil || want.FirewallIPv4.Name != old.FirewallIPv4.Name) { - firewall, err := fakeGCE.GetFirewall(old.FirewallIPv4.Name) - if utils.IgnoreHTTPNotFound(err) != nil { - t.Errorf("fakeGCE.GetFirewall() returned error %v", err) - } - if firewall != nil { - t.Errorf("found firewall %v, which should have been deleted", firewall.Name) - } - } - - if old.FwdRuleIPv6 != nil && (want.FwdRuleIPv6 == nil || want.FwdRuleIPv6.Name != old.FwdRuleIPv6.Name) { - fwdRule, err := fakeGCE.GetRegionForwardingRule(old.FwdRuleIPv6.Name, old.FwdRuleIPv6.Region) - if utils.IgnoreHTTPNotFound(err) != nil { - t.Errorf("fakeGCE.GetForwardingRule() returned error %v", err) - } - if fwdRule != nil { - t.Errorf("found forwarding rule %v, which should have been deleted", fwdRule.Name) - } - } - - if old.FirewallIPv6 != nil && (want.FirewallIPv6 == nil || want.FirewallIPv6.Name != old.FirewallIPv6.Name) { - firewall, err := fakeGCE.GetFirewall(old.FirewallIPv6.Name) - if utils.IgnoreHTTPNotFound(err) != nil { - t.Errorf("fakeGCE.GetFirewall() returned error %v", err) - } - if firewall != nil { - t.Errorf("found firewall %v, which should have been deleted", firewall.Name) - } - } - - if old.HC != nil && (want.HC == nil || want.HC.Name != old.HC.Name) { - hc, err := fakeGCE.GetHealthCheck(old.HC.Name) - if utils.IgnoreHTTPNotFound(err) != nil { - t.Errorf("fakeGCE.GetHealthCheck() returned error %v", err) - } - if hc != nil { - t.Errorf("found health check %v, which should have been deleted", hc.Name) - } - } - - // We don't need to clean up firewall for health checks, since they are shared + mixedprotocoltest.VerifyResourcesExist(t, fakeGCE, want) + mixedprotocoltest.VerifyResourcesCleanedUp(t, fakeGCE, old, want) } diff --git a/pkg/loadbalancers/l4netlb.go b/pkg/loadbalancers/l4netlb.go index 4fedd757a2..1e04b61974 100644 --- a/pkg/loadbalancers/l4netlb.go +++ b/pkg/loadbalancers/l4netlb.go @@ -62,10 +62,12 @@ type L4NetLB struct { scope meta.KeyType namer namer.L4ResourcesNamer // recorder is used to generate k8s Events. - recorder record.EventRecorder - Service *corev1.Service - NamespacedName types.NamespacedName - healthChecks healthchecksl4.L4HealthChecks + recorder record.EventRecorder + Service *corev1.Service + NamespacedName types.NamespacedName + healthChecks healthchecksl4.L4HealthChecks + // mixedManager is responsible for managing forwarding rules for mixed protocol lbs + mixedManager *forwardingrules.MixedManagerNetLB forwardingRules ForwardingRulesProvider enableDualStack bool // represents if `enable strong session affinity` flag was set @@ -135,6 +137,15 @@ type L4NetLBParams struct { // NewL4NetLB creates a new Handler for the given L4NetLB service. func NewL4NetLB(params *L4NetLBParams, logger klog.Logger) *L4NetLB { logger = logger.WithName("L4NetLBHandler") + forwardingRulesProvider := forwardingrules.New(params.Cloud, meta.VersionGA, meta.Regional, logger) + mixedManager := &forwardingrules.MixedManagerNetLB{ + Namer: params.Namer, + Provider: forwardingRulesProvider, + Recorder: params.Recorder, + Logger: logger, + Cloud: params.Cloud, + Service: params.Service, + } l4netlb := &L4NetLB{ cloud: params.Cloud, scope: meta.Regional, @@ -144,7 +155,8 @@ func NewL4NetLB(params *L4NetLBParams, logger klog.Logger) *L4NetLB { NamespacedName: types.NamespacedName{Name: params.Service.Name, Namespace: params.Service.Namespace}, backendPool: backends.NewPoolWithConnectionTrackingPolicy(params.Cloud, params.Namer, params.StrongSessionAffinityEnabled), healthChecks: healthchecksl4.NewL4HealthChecks(params.Cloud, params.Recorder, logger), - forwardingRules: forwardingrules.New(params.Cloud, meta.VersionGA, meta.Regional, logger), + forwardingRules: forwardingRulesProvider, + mixedManager: mixedManager, enableDualStack: params.DualStackEnabled, enableStrongSessionAffinity: params.StrongSessionAffinityEnabled, networkResolver: params.NetworkResolver, @@ -332,7 +344,11 @@ func (l4netlb *L4NetLB) connectionTrackingPolicy() *composite.BackendServiceConn func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcLink string) string { bsName := l4netlb.namer.L4Backend(l4netlb.Service.Namespace, l4netlb.Service.Name) servicePorts := l4netlb.Service.Spec.Ports - protocol := utils.GetProtocol(servicePorts) + + protocol := string(utils.GetProtocol(servicePorts)) + if l4netlb.enableMixedProtocol { + protocol = backends.GetProtocol(servicePorts) + } localityLbPolicy := l4netlb.determineBackendServiceLocalityPolicy() @@ -340,7 +356,7 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL backendParams := backends.L4BackendServiceParams{ Name: bsName, HealthCheckLink: hcLink, - Protocol: string(protocol), + Protocol: protocol, SessionAffinity: string(l4netlb.Service.Spec.SessionAffinity), Scheme: string(cloud.SchemeExternal), NamespacedName: l4netlb.NamespacedName, @@ -385,6 +401,11 @@ func (l4netlb *L4NetLB) ensureDualStackResources(result *L4NetLBSyncResult, node // - IPv4 Forwarding Rule // - IPv4 Firewall func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) { + if l4netlb.enableMixedProtocol && forwardingrules.NeedsMixed(l4netlb.Service.Spec.Ports) { + l4netlb.ensureIPv4MixedResources(result, nodeNames, bsLink) + return + } + fr, ipAddrType, wasUpdate, err := l4netlb.ensureIPv4ForwardingRule(bsLink) result.GCEResourceUpdate.SetForwardingRule(wasUpdate) if err != nil { @@ -411,6 +432,43 @@ func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames result.Status = utils.AddIPToLBStatus(result.Status, fr.IPAddress) } +func (l4netlb *L4NetLB) ensureIPv4MixedResources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) { + res, err := l4netlb.mixedManager.EnsureIPv4(bsLink) + + result.GCEResourceUpdate.SetForwardingRule(res.SyncStatus) + if err != nil { + // User can misconfigure the forwarding rule if Network Tier will not match service level Network Tier. + result.GCEResourceInError = annotations.ForwardingRuleResource + result.Error = fmt.Errorf("failed to ensure mixed protocol forwarding rules - %w", err) + result.MetricsLegacyState.IsUserError = utils.IsUserError(err) + return + } + + var ipAddr string + if res.UDPFwdRule != nil { + result.Annotations[annotations.UDPForwardingRuleKey] = res.UDPFwdRule.Name + if res.TCPFwdRule.NetworkTier == cloud.NetworkTierPremium.ToGCEValue() { + result.MetricsLegacyState.IsPremiumTier = true + } + ipAddr = res.UDPFwdRule.IPAddress + } + if res.TCPFwdRule != nil { + result.Annotations[annotations.TCPForwardingRuleKey] = res.TCPFwdRule.Name + if res.TCPFwdRule.NetworkTier == cloud.NetworkTierPremium.ToGCEValue() { + result.MetricsLegacyState.IsPremiumTier = true + } + ipAddr = res.TCPFwdRule.IPAddress + } + + l4netlb.ensureIPv4NodesFirewall(nodeNames, ipAddr, result) + if result.Error != nil { + l4netlb.svcLogger.Error(err, "ensureIPv4MixedResources: Failed to ensure nodes firewall for L4 NetLB Service") + return + } + + result.Status = utils.AddIPToLBStatus(result.Status, ipAddr) +} + func (l4netlb *L4NetLB) ensureIPv4NodesFirewall(nodeNames []string, ipAddress string, result *L4NetLBSyncResult) { // DisableL4LBFirewall flag disables L4 FW enforcment to remove conflicts with firewall policies if l4netlb.disableNodesFirewallProvisioning { @@ -525,6 +583,9 @@ func (l4netlb *L4NetLB) deleteIPv4ResourcesAnnotationBased(result *L4NetLBSyncRe result.Error = err result.GCEResourceInError = annotations.ForwardingRuleResource } + if err = l4netlb.mixedManager.DeleteIPv4(); err != nil { + l4netlb.svcLogger.Error(err, "Failed to delete mixed protocol forwarding rules for NetLB RBS service") + } } // Deleting non-existent address do not print error audit logs, and we don't store address in annotations diff --git a/pkg/loadbalancers/l4netlb_mixed_test.go b/pkg/loadbalancers/l4netlb_mixed_test.go new file mode 100644 index 0000000000..3cfe0e9db3 --- /dev/null +++ b/pkg/loadbalancers/l4netlb_mixed_test.go @@ -0,0 +1,257 @@ +package loadbalancers + +import ( + "context" + "testing" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/mock" + "github.com/google/go-cmp/cmp" + "google.golang.org/api/compute/v1" + api_v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "k8s.io/cloud-provider-gcp/providers/gce" + "k8s.io/ingress-gce/pkg/flags" + "k8s.io/ingress-gce/pkg/healthchecksl4" + "k8s.io/ingress-gce/pkg/loadbalancers/mixedprotocoltest" + "k8s.io/ingress-gce/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest" + "k8s.io/ingress-gce/pkg/network" + "k8s.io/ingress-gce/pkg/utils/namer" + "k8s.io/klog/v2" +) + +// TestEnsureMixedNetLB tests transitions between mixed and single protocol ILBs +// Steps: +// 1. Arrange: +// a) Create existing GCE resources with fakeGCE based on starting state. +// b) Create L4 struct with mixed protocol feature flag enabled +// 2. Act: +// a) Run l4.Ensure(service) +// 3. Assert: +// a) Verify result +// b) Verify resources from fakeGCE with resources in the end state +// c) Verify resources from starting state not used in end state are cleaned up +func TestEnsureMixedNetLB(t *testing.T) { + startState := []struct { + desc string + // have + resources mixedprotocoltest.GCEResources + annotations map[string]string + ingress []api_v1.LoadBalancerIngress + }{ + { + desc: "nothing", + }, + { + desc: "ipv4 tcp", + resources: mixedprotocolnetlbtest.TCPResources(), + annotations: mixedprotocolnetlbtest.AnnotationsTCP(), + ingress: mixedprotocolnetlbtest.IPv4Ingress(), + }, + { + desc: "ipv4 udp", + resources: mixedprotocolnetlbtest.UDPResources(), + annotations: mixedprotocolnetlbtest.AnnotationsTCP(), + ingress: mixedprotocolnetlbtest.IPv4Ingress(), + }, + { + desc: "ipv4 mixed", + resources: mixedprotocolnetlbtest.MixedResources(), + annotations: mixedprotocolnetlbtest.AnnotationsMixed(), + ingress: mixedprotocolnetlbtest.IPv4Ingress(), + }, + } + + endState := []struct { + desc string + // have + spec api_v1.ServiceSpec + // want + resources mixedprotocoltest.GCEResources + annotations map[string]string + }{ + { + desc: "ipv4 tcp", + spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, nil), + annotations: mixedprotocolnetlbtest.AnnotationsTCP(), + resources: mixedprotocolnetlbtest.TCPResources(), + }, + { + desc: "ipv4 udp", + spec: mixedprotocoltest.SpecIPv4(nil, []int32{53}), + annotations: mixedprotocolnetlbtest.AnnotationsUDP(), + resources: mixedprotocolnetlbtest.UDPResources(), + }, + { + desc: "ipv4 mixed", + spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, []int32{53}), + annotations: mixedprotocolnetlbtest.AnnotationsMixed(), + resources: mixedprotocolnetlbtest.MixedResources(), + }, + } + + // this flag is for single protocol only, mixed protocol use DiscretePortForwarding by default + flags.F.EnableDiscretePortForwarding = true + for _, s := range startState { + for _, e := range endState { + desc := s.desc + " -> " + e.desc + t.Run(desc, func(t *testing.T) { + t.Parallel() + svc := &api_v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + UID: mixedprotocoltest.TestUID, + Name: mixedprotocoltest.TestName, + Namespace: mixedprotocoltest.TestNamespace, + Annotations: s.annotations, + }, + Spec: e.spec, + Status: api_v1.ServiceStatus{ + LoadBalancer: api_v1.LoadBalancerStatus{ + Ingress: s.ingress, + }, + }, + } + l4netlb, fakeGCE := arrangeNetLB(t, s.resources, svc) + + result := l4netlb.EnsureFrontend([]string{mixedprotocoltest.TestNode}, svc) + + wantResult := &L4NetLBSyncResult{ + Annotations: e.annotations, + SyncType: "create", + } + if s.resources.BackendService != nil { + wantResult.SyncType = "update" + } + + assertNetLBResult(t, result, wantResult) + assertResources(t, fakeGCE, e.resources, s.resources) + }) + } + } +} + +func assertNetLBResult(t *testing.T, got, want *L4NetLBSyncResult) { + t.Helper() + + if got.Error != want.Error { + t.Errorf("got.Error != want.Error: got %v, want %v", got.Error, want.Error) + } + if got.SyncType != want.SyncType { + t.Errorf("got.SyncType != want.SyncType: got %v, want %v", got.SyncType, want.SyncType) + } + + if diff := cmp.Diff(got.Annotations, want.Annotations); diff != "" { + t.Errorf("got.Annotations != want.Annotations: (-got +want):\n%s", diff) + } +} + +func arrangeNetLB(t *testing.T, existing mixedprotocoltest.GCEResources, svc *api_v1.Service) (*L4NetLB, *gce.Cloud) { + t.Helper() + vals := gce.DefaultTestClusterValues() + fakeGCE := gce.NewFakeGCECloud(vals) + + // Update hooks to be able to perform updates + mockGCE := fakeGCE.Compute().(*cloud.MockGCE) + mockGCE.MockRegionBackendServices.UpdateHook = mock.UpdateRegionBackendServiceHook + mockGCE.MockRegionBackendServices.PatchHook = mock.UpdateRegionBackendServiceHook + mockGCE.MockFirewalls.UpdateHook = mock.UpdateFirewallHook + mockGCE.MockFirewalls.PatchHook = mock.UpdateFirewallHook + + namer := namer.NewL4Namer(kubeSystemUID, nil) + params := &L4NetLBParams{ + Service: svc, + Cloud: fakeGCE, + Namer: namer, + Recorder: &record.FakeRecorder{}, + NetworkResolver: network.NewFakeResolver(network.DefaultNetwork(fakeGCE)), + EnableMixedProtocol: true, + DualStackEnabled: true, + } + l4NetLB := NewL4NetLB(params, klog.TODO()) + l4NetLB.healthChecks = healthchecksl4.Fake(fakeGCE, params.Recorder) + + if err := fakeGCE.InsertInstance(vals.ProjectID, vals.ZoneName, &compute.Instance{ + Name: mixedprotocoltest.TestNode, + Tags: &compute.Tags{Items: []string{mixedprotocoltest.TestNode}}, + }); err != nil { + t.Errorf("fakeGCE.InsertInstance() returned error %v", err) + } + + dualStackSubnetwork := &compute.Subnetwork{ + StackType: "IPV4_IPV6", + Ipv6AccessType: "EXTERNAL", + } + if err := fakeGCE.Compute().Subnetworks().Insert(context.TODO(), meta.RegionalKey("", vals.Region), dualStackSubnetwork); err != nil { + t.Errorf("fakeGCE.Compute().Subnetworks().Insert() returned error %v", err) + } + + if err := existing.Create(fakeGCE); err != nil { + t.Errorf("existing.Create() returned error %v", err) + } + + return l4NetLB, fakeGCE +} + +func TestDeleteMixedNetLB(t *testing.T) { + testCases := []struct { + desc string + resources mixedprotocoltest.GCEResources + spec api_v1.ServiceSpec + annotations map[string]string + ingress []api_v1.LoadBalancerIngress + }{ + { + desc: "ipv4 tcp", + resources: mixedprotocolnetlbtest.TCPResources(), + spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, nil), + annotations: mixedprotocolnetlbtest.AnnotationsTCP(), + ingress: mixedprotocolnetlbtest.IPv4Ingress(), + }, + { + desc: "ipv4 udp", + resources: mixedprotocolnetlbtest.UDPResources(), + spec: mixedprotocoltest.SpecIPv4(nil, []int32{53}), + annotations: mixedprotocolnetlbtest.AnnotationsTCP(), + ingress: mixedprotocolnetlbtest.IPv4Ingress(), + }, + { + desc: "ipv4 mixed", + resources: mixedprotocolnetlbtest.MixedResources(), + spec: mixedprotocoltest.SpecIPv4([]int32{80, 443}, []int32{53}), + annotations: mixedprotocolnetlbtest.AnnotationsMixed(), + ingress: mixedprotocolnetlbtest.IPv4Ingress(), + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + svc := &api_v1.Service{ + ObjectMeta: meta_v1.ObjectMeta{ + UID: mixedprotocoltest.TestUID, + Name: mixedprotocoltest.TestName, + Namespace: mixedprotocoltest.TestNamespace, + Annotations: tc.annotations, + }, + Spec: tc.spec, + Status: api_v1.ServiceStatus{ + LoadBalancer: api_v1.LoadBalancerStatus{ + Ingress: tc.ingress, + }, + }, + } + l4NetLB, fakeGCE := arrangeNetLB(t, tc.resources, svc) + + result := l4NetLB.EnsureLoadBalancerDeleted(svc) + + wantResult := &L4NetLBSyncResult{ + Annotations: map[string]string{}, + SyncType: "delete", + } + assertNetLBResult(t, result, wantResult) + mixedprotocoltest.VerifyResourcesCleanedUp(t, fakeGCE, tc.resources, mixedprotocoltest.GCEResources{}) + }) + } +} diff --git a/pkg/loadbalancers/mixedprotocoltest/consts.go b/pkg/loadbalancers/mixedprotocoltest/consts.go index 92a28e112f..179216cc99 100644 --- a/pkg/loadbalancers/mixedprotocoltest/consts.go +++ b/pkg/loadbalancers/mixedprotocoltest/consts.go @@ -7,8 +7,6 @@ const ( TestName = "test-name" // TestNamespace is the name of the Kubernetes namespace used for mixed protocol tests TestNamespace = "test-namespace" - // IPv4Address is the internal IPv4 address used for mixed protocol tests - IPv4Address = "10.0.0.10" - // IPv6Address is the internal IPv6 address used for mixed protocol tests - IPv6Address = "fd20:dd1:549d:7000:0:b:0:0" + // TestUID is the UID for services. It is used by legacy naming scheme to generate name + TestUID = "1234567890" ) diff --git a/pkg/loadbalancers/mixedprotocoltest/annotations.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/annotations.go similarity index 99% rename from pkg/loadbalancers/mixedprotocoltest/annotations.go rename to pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/annotations.go index c71f62dc00..d7af299e70 100644 --- a/pkg/loadbalancers/mixedprotocoltest/annotations.go +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/annotations.go @@ -1,4 +1,4 @@ -package mixedprotocoltest +package mixedprotocolilbtest // AnnotationsTCP returns annotations for TCP ILB func AnnotationsTCP() map[string]string { diff --git a/pkg/loadbalancers/mixedprotocoltest/ingress.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/ingress.go similarity index 96% rename from pkg/loadbalancers/mixedprotocoltest/ingress.go rename to pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/ingress.go index 00e8529c00..9129da7b36 100644 --- a/pkg/loadbalancers/mixedprotocoltest/ingress.go +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/ingress.go @@ -1,4 +1,4 @@ -package mixedprotocoltest +package mixedprotocolilbtest import ( api_v1 "k8s.io/api/core/v1" diff --git a/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/ips.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/ips.go new file mode 100644 index 0000000000..124fae8c95 --- /dev/null +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/ips.go @@ -0,0 +1,9 @@ +// Package mixedprotocolilbtests contains helpers specific for testing internal mixed protocol L4 load balancers +package mixedprotocolilbtest + +const ( + // IPv4Address is the internal IPv4 address used for mixed protocol tests + IPv4Address = "10.0.0.10" + // IPv6Address is the internal IPv6 address used for mixed protocol tests + IPv6Address = "fd20:dd1:549d:7000:0:b:0:0" +) diff --git a/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/resources.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/resources.go new file mode 100644 index 0000000000..74e3bc95cf --- /dev/null +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolilbtest/resources.go @@ -0,0 +1,300 @@ +package mixedprotocolilbtest + +import ( + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "google.golang.org/api/compute/v1" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/loadbalancers/mixedprotocoltest" +) + +// TCPResources returns GCE resources for a TCP IPv4 ILB that listens on ports 80 and 443 +func TCPResources() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleTCPIPv4Name: ForwardingRuleTCP([]string{"80", "443"}), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("TCP"), + } +} + +// UDPResources returns GCE resources for an UDP IPv4 ILB that listens on port 53 +func UDPResources() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleUDPIPv4Name: ForwardingRuleUDP([]string{"53"}), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UDP"), + } +} + +// L3Resources returns GCE resources for a mixed protocol IPv4 ILB, that listens on tcp:80, tcp:443 and udp:53 +func L3Resources() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleL3IPv4Name: ForwardingRuleL3(), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UNSPECIFIED"), + } +} + +// TCPResourcesIPv6 returns GCE resources for a TCP IPv6 ILB that listens on ports 80 and 443 +func TCPResourcesIPv6() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleTCPIPv6Name: ForwardingrukeRuleTCPIPv6([]string{"80", "443"}), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv6Name: mixedprotocoltest.FirewallIPv6([]*compute.FirewallAllowed{ + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv6Name: mixedprotocoltest.HealthCheckFirewallIPv6(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("TCP"), + } +} + +// UDPResourcesIPv6 returns GCE resources for an UDP IPv6 ILB that listens on port 53 +func UDPResourcesIPv6() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleUDPIPv6Name: ForwardingRuleUDPIPv6([]string{"53"}), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv6Name: mixedprotocoltest.FirewallIPv6([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv6Name: mixedprotocoltest.HealthCheckFirewallIPv6(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UDP"), + } +} + +// L3ResourcesIPv6 returns GCE resources for a mixed protocol IPv6 ILB, that listens on tcp:80, tcp:443 and udp:53 +func L3ResourcesIPv6() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleL3IPv6Name: ForwardingRuleL3IPv6(), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv6Name: mixedprotocoltest.FirewallIPv6([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv6Name: mixedprotocoltest.HealthCheckFirewallIPv6(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UNSPECIFIED"), + } +} + +// TCPResourcesDualStack returns GCE resources for a TCP dual stack ILB that listens on ports 80 and 443 +func TCPResourcesDualStack() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleTCPIPv4Name: ForwardingRuleTCP([]string{"80", "443"}), + mixedprotocoltest.ForwardingRuleTCPIPv6Name: ForwardingrukeRuleTCPIPv6([]string{"80", "443"}), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.FirewallIPv6Name: mixedprotocoltest.FirewallIPv6([]*compute.FirewallAllowed{ + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + mixedprotocoltest.HealthCheckFirewallIPv6Name: mixedprotocoltest.HealthCheckFirewallIPv6(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("TCP"), + } +} + +// UDPResourcesDualStack returns GCE resources for an UDP dual stack ILB that listens on port 53 +func UDPResourcesDualStack() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleUDPIPv4Name: ForwardingRuleUDP([]string{"53"}), + mixedprotocoltest.ForwardingRuleUDPIPv6Name: ForwardingRuleUDPIPv6([]string{"53"}), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + }), + mixedprotocoltest.FirewallIPv6Name: mixedprotocoltest.FirewallIPv6([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + mixedprotocoltest.HealthCheckFirewallIPv6Name: mixedprotocoltest.HealthCheckFirewallIPv6(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UDP"), + } +} + +// L3ResourcesDualStack returns GCE resources for a mixed protocol dual stack ILB, that listens on tcp:80, tcp:443 and udp:53 +func L3ResourcesDualStack() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleL3IPv4Name: ForwardingRuleL3(), + mixedprotocoltest.ForwardingRuleL3IPv6Name: ForwardingRuleL3IPv6(), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.FirewallIPv6Name: mixedprotocoltest.FirewallIPv6([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + mixedprotocoltest.HealthCheckFirewallIPv6Name: mixedprotocoltest.HealthCheckFirewallIPv6(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UNSPECIFIED"), + } +} + +// ForwardingRuleL3 returns an L3 Forwarding Rule for ILB +func ForwardingRuleL3() *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: mixedprotocoltest.ForwardingRuleL3IPv4Name, + Region: "us-central1", + IPProtocol: "L3_DEFAULT", + AllPorts: true, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "INTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// ForwardingRuleL3IPv6 returns an L3 Forwarding Rule for ILB +func ForwardingRuleL3IPv6() *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: mixedprotocoltest.ForwardingRuleL3IPv6Name, + Region: "us-central1", + IPProtocol: "L3_DEFAULT", + IpVersion: "IPV6", + AllPorts: true, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "INTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name"}`, + } +} + +// ForwardingRuleUDP returns a UDP Forwarding Rule with specified ports +func ForwardingRuleUDP(ports []string) *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: mixedprotocoltest.ForwardingRuleUDPIPv4Name, + Region: "us-central1", + IPProtocol: "UDP", + Ports: ports, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "INTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// ForwardingRuleUDPIPv6 returns a UDP Forwarding Rule with specified ports +func ForwardingRuleUDPIPv6(ports []string) *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: mixedprotocoltest.ForwardingRuleUDPIPv6Name, + Region: "us-central1", + IPProtocol: "UDP", + IpVersion: "IPV6", + Ports: ports, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "INTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name"}`, + } +} + +// ForwardingRuleTCP returns a TCP Forwarding Rule with specified ports +func ForwardingRuleTCP(ports []string) *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: mixedprotocoltest.ForwardingRuleTCPIPv4Name, + Region: "us-central1", + IPProtocol: "TCP", + Ports: ports, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "INTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// ForwardingrukeRuleTCPIPv6 returns a TCP Forwarding Rule with specified ports +func ForwardingrukeRuleTCPIPv6(ports []string) *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: mixedprotocoltest.ForwardingRuleTCPIPv6Name, + Region: "us-central1", + IPProtocol: "TCP", + IpVersion: "IPV6", + Ports: ports, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "INTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name"}`, + } +} + +// BackendService returns Backend Service for ILB +// protocol should be set to: +// - `UNSPECIFIED` for mixed protocol (L3) +// - `TCP` for TCP only +// - `UDP` for UDP only +func BackendService(protocol string) *compute.BackendService { + return &compute.BackendService{ + Name: mixedprotocoltest.BackendServiceName, + Region: "us-central1", + Protocol: protocol, + SessionAffinity: "NONE", + LoadBalancingScheme: "INTERNAL", + HealthChecks: []string{"https://www.googleapis.com/compute/v1/projects/test-project/global/healthChecks/k8s2-axyqjz2d-l4-shared-hc"}, + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// HealthCheck returns shared HealthCheck +func HealthCheck() *composite.HealthCheck { + return &composite.HealthCheck{ + Name: mixedprotocoltest.HealthCheckName, + CheckIntervalSec: 8, + TimeoutSec: 1, + HealthyThreshold: 1, + UnhealthyThreshold: 3, + Type: "HTTP", + Version: meta.VersionGA, + Scope: meta.Global, + HttpHealthCheck: &composite.HTTPHealthCheck{Port: 10256, RequestPath: "/healthz"}, + Description: `{"networking.gke.io/service-name":"","networking.gke.io/api-version":"ga","networking.gke.io/resource-description":"This resource is shared by all L4 ILB Services using ExternalTrafficPolicy: Cluster."}`, + } +} diff --git a/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/annotations.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/annotations.go new file mode 100644 index 0000000000..46c77d7bd8 --- /dev/null +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/annotations.go @@ -0,0 +1,113 @@ +package mixedprotocolnetlbtest + +// AnnotationsTCP returns annotations for TCP NetLB +func AnnotationsTCP() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc": "k8s2-axyqjz2d-l4-shared-hc-fw", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/tcp-forwarding-rule": "a1234567890", + "service.kubernetes.io/firewall-rule": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + } +} + +// AnnotationsTCPIPv6 returns annotations for TCP NetLB +func AnnotationsTCPIPv6() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc-ipv6": "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/tcp-forwarding-rule-ipv6": "a1234567890-ipv6", + "service.kubernetes.io/firewall-rule-ipv6": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + } +} + +// AnnotationsTCPDualStack returns annotations for TCP NetLB +func AnnotationsTCPDualStack() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc": "k8s2-axyqjz2d-l4-shared-hc-fw", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/tcp-forwarding-rule": "a1234567890", + "service.kubernetes.io/firewall-rule": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/firewall-rule-for-hc-ipv6": "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + "service.kubernetes.io/tcp-forwarding-rule-ipv6": "a1234567890-ipv6", + "service.kubernetes.io/firewall-rule-ipv6": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + } +} + +// AnnotationsUDP returns annotations for UDP NetLB +func AnnotationsUDP() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc": "k8s2-axyqjz2d-l4-shared-hc-fw", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/udp-forwarding-rule": "a1234567890", + "service.kubernetes.io/firewall-rule": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + } +} + +// AnnotationsUDPIPv6 returns annotations for UDP NetLB +func AnnotationsUDPIPv6() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc-ipv6": "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/udp-forwarding-rule-ipv6": "a1234567890-ipv6", + "service.kubernetes.io/firewall-rule-ipv6": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + } +} + +// AnnotationsUDPDualStack returns annotations for UDP NetLB +func AnnotationsUDPDualStack() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc": "k8s2-axyqjz2d-l4-shared-hc-fw", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/udp-forwarding-rule": "a1234567890", + "service.kubernetes.io/firewall-rule": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/firewall-rule-for-hc-ipv6": "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + "service.kubernetes.io/udp-forwarding-rule-ipv6": "a1234567890-ipv6", + "service.kubernetes.io/firewall-rule-ipv6": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + } +} + +// AnnotationsMixed returns annotations for mixed protocol NetLB +func AnnotationsMixed() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc": "k8s2-axyqjz2d-l4-shared-hc-fw", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/udp-forwarding-rule": "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/tcp-forwarding-rule": "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/firewall-rule": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + } +} + +// AnnotationsMixedIPv6 returns annotations for mixed protocol NetLB +func AnnotationsMixedIPv6() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc-ipv6": "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/udp-forwarding-rule-ipv6": "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + "service.kubernetes.io/tcp-forwarding-rule-ipv6": "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + "service.kubernetes.io/firewall-rule-ipv6": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + } +} + +// AnnotationsMixedDualStack returns annotations for dual stack NetLB +func AnnotationsMixedDualStack() map[string]string { + return map[string]string{ + "service.kubernetes.io/healthcheck": "k8s2-axyqjz2d-l4-shared-hc", + "service.kubernetes.io/firewall-rule-for-hc": "k8s2-axyqjz2d-l4-shared-hc-fw", + "service.kubernetes.io/backend-service": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/udp-forwarding-rule": "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/tcp-forwarding-rule": "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/firewall-rule": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + "service.kubernetes.io/firewall-rule-for-hc-ipv6": "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + "service.kubernetes.io/udp-forwarding-rule-ipv6": "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + "service.kubernetes.io/tcp-forwarding-rule-ipv6": "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + "service.kubernetes.io/firewall-rule-ipv6": "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + } +} diff --git a/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/ingress.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/ingress.go new file mode 100644 index 0000000000..3f2f62e01d --- /dev/null +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/ingress.go @@ -0,0 +1,16 @@ +package mixedprotocolnetlbtest + +import ( + api_v1 "k8s.io/api/core/v1" +) + +// IPv4Ingress is an Ingress for already existing IPv4 load balancers +func IPv4Ingress() []api_v1.LoadBalancerIngress { + mode := api_v1.LoadBalancerIPModeVIP + return []api_v1.LoadBalancerIngress{ + { + IP: IPv4Address, + IPMode: &mode, + }, + } +} diff --git a/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/ips.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/ips.go new file mode 100644 index 0000000000..1b0bd87a4f --- /dev/null +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/ips.go @@ -0,0 +1,7 @@ +// Package mixedprotocolnetlbtests contains helpers specific for testing external mixed protocol L4 load balancers +package mixedprotocolnetlbtest + +const ( + // IPv4Address is the IPv4 address used for mixed protocol tests + IPv4Address = "34.122.234.156" +) diff --git a/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/resources.go b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/resources.go new file mode 100644 index 0000000000..62c8159ff7 --- /dev/null +++ b/pkg/loadbalancers/mixedprotocoltest/mixedprotocolnetlbtest/resources.go @@ -0,0 +1,131 @@ +package mixedprotocolnetlbtest + +import ( + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "google.golang.org/api/compute/v1" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/loadbalancers/mixedprotocoltest" +) + +// TCPResources returns GCE resources for a TCP IPv4 NetLB that listens on ports 80 and 443 +func TCPResources() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleLegacyName: ForwardingRuleTCP( + mixedprotocoltest.ForwardingRuleLegacyName, []string{"80", "443"}, + ), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("TCP"), + } +} + +// UDPResources returns GCE resources for an UDP IPv4 NetLB that listens on port 53 +func UDPResources() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleLegacyName: ForwardingRuleUDP( + mixedprotocoltest.ForwardingRuleLegacyName, []string{"53"}, + ), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UDP"), + } +} + +// MixedResources returns GCE resources for a mixed protocol IPv4 NetLB, that listens on tcp:80, tcp:443 and udp:53 +func MixedResources() mixedprotocoltest.GCEResources { + return mixedprotocoltest.GCEResources{ + ForwardingRules: map[string]*compute.ForwardingRule{ + mixedprotocoltest.ForwardingRuleTCPIPv4Name: ForwardingRuleTCP( + mixedprotocoltest.ForwardingRuleTCPIPv4Name, []string{"80", "443"}, + ), + mixedprotocoltest.ForwardingRuleUDPIPv4Name: ForwardingRuleUDP( + mixedprotocoltest.ForwardingRuleUDPIPv4Name, []string{"53"}, + ), + }, + Firewalls: map[string]*compute.Firewall{ + mixedprotocoltest.FirewallIPv4Name: mixedprotocoltest.Firewall([]*compute.FirewallAllowed{ + {IPProtocol: "udp", Ports: []string{"53"}}, + {IPProtocol: "tcp", Ports: []string{"80", "443"}}, + }), + mixedprotocoltest.HealthCheckFirewallIPv4Name: mixedprotocoltest.HealthCheckFirewall(), + }, + HealthCheck: HealthCheck(), + BackendService: BackendService("UNSPECIFIED"), + } +} + +// ForwardingRuleUDP returns a UDP Forwarding Rule with specified ports +func ForwardingRuleUDP(name string, ports []string) *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: name, + Region: "us-central1", + IPProtocol: "UDP", + Ports: ports, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "EXTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// ForwardingRuleTCP returns a TCP Forwarding Rule with specified ports +func ForwardingRuleTCP(name string, ports []string) *compute.ForwardingRule { + return &compute.ForwardingRule{ + Name: name, + Region: "us-central1", + IPProtocol: "TCP", + Ports: ports, + BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + LoadBalancingScheme: "EXTERNAL", + NetworkTier: "PREMIUM", + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// BackendService returns Backend Service for NetLB +// protocol should be set to: +// - `UNSPECIFIED` for mixed protocol (L3) +// - `TCP` for TCP only +// - `UDP` for UDP only +func BackendService(protocol string) *compute.BackendService { + return &compute.BackendService{ + Name: "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + Region: "us-central1", + Protocol: protocol, + SessionAffinity: "NONE", + LoadBalancingScheme: "EXTERNAL", + HealthChecks: []string{"https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/healthChecks/k8s2-axyqjz2d-l4-shared-hc"}, + Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, + } +} + +// HealthCheck returns shared HealthCheck +func HealthCheck() *composite.HealthCheck { + return &composite.HealthCheck{ + Name: mixedprotocoltest.HealthCheckName, + CheckIntervalSec: 8, + TimeoutSec: 1, + HealthyThreshold: 1, + UnhealthyThreshold: 3, + Type: "HTTP", + Region: "us-central1", + Scope: meta.Regional, + Version: meta.VersionGA, + HttpHealthCheck: &composite.HTTPHealthCheck{Port: 10256, RequestPath: "/healthz"}, + Description: `{"networking.gke.io/service-name":"","networking.gke.io/api-version":"ga","networking.gke.io/resource-description":"This resource is shared by all L4 XLB Services using ExternalTrafficPolicy: Cluster."}`, + } +} diff --git a/pkg/loadbalancers/mixedprotocoltest/resources.go b/pkg/loadbalancers/mixedprotocoltest/resources.go index 179370106a..88460718d7 100644 --- a/pkg/loadbalancers/mixedprotocoltest/resources.go +++ b/pkg/loadbalancers/mixedprotocoltest/resources.go @@ -1,175 +1,189 @@ +// Package mixedprotocoltest contains common helpers for testing mixed protocol L4 load balancers package mixedprotocoltest import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/api/compute/v1" + "k8s.io/cloud-provider-gcp/providers/gce" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/healthchecksprovider" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/klog/v2" ) -// GCEResources collects all GCE resources that are managed by the mixed protocol ILB -type GCEResources struct { - FwdRuleIPv4 *compute.ForwardingRule - FirewallIPv4 *compute.Firewall - FwdRuleIPv6 *compute.ForwardingRule - FirewallIPv6 *compute.Firewall - HC *compute.HealthCheck - HCFirewallIPv4 *compute.Firewall - HCFirewallIPv6 *compute.Firewall - BS *compute.BackendService -} - -// TCPResources returns GCE resources for a TCP IPv4 ILB that listens on ports 80 and 443 -func TCPResources() GCEResources { - return GCEResources{ - FwdRuleIPv4: ForwardingRuleTCP([]string{"80", "443"}), - FirewallIPv4: Firewall([]*compute.FirewallAllowed{ - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - BS: BackendService("TCP"), - HC: HealthCheck(), - HCFirewallIPv4: HealthCheckFirewall(), - } -} - -// UDPResources returns GCE resources for an UDP IPv4 ILB that listens on port 53 -func UDPResources() GCEResources { - return GCEResources{ - FwdRuleIPv4: ForwardingRuleUDP([]string{"53"}), - FirewallIPv4: Firewall([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - }), - BS: BackendService("UDP"), - HC: HealthCheck(), - HCFirewallIPv4: HealthCheckFirewall(), - } -} - -// L3Resources returns GCE resources for a mixed protocol IPv4 ILB, that listens on tcp:80, tcp:443 and udp:53 -func L3Resources() GCEResources { - return GCEResources{ - FwdRuleIPv4: ForwardingRuleL3(), - FirewallIPv4: Firewall([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - BS: BackendService("UNSPECIFIED"), - HC: HealthCheck(), - HCFirewallIPv4: HealthCheckFirewall(), - } -} - -// TCPResourcesIPv6 returns GCE resources for a TCP IPv6 ILB that listens on ports 80 and 443 -func TCPResourcesIPv6() GCEResources { - return GCEResources{ - FwdRuleIPv6: ForwardingrukeRuleTCPIPv6([]string{"80", "443"}), - FirewallIPv6: FirewallIPv6([]*compute.FirewallAllowed{ - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - BS: BackendService("TCP"), - HC: HealthCheck(), - HCFirewallIPv6: HealthCheckFirewallIPv6(), - } -} - -// UDPResourcesIPv6 returns GCE resources for an UDP IPv6 ILB that listens on port 53 -func UDPResourcesIPv6() GCEResources { - return GCEResources{ - FwdRuleIPv6: ForwardingRuleUDPIPv6([]string{"53"}), - FirewallIPv6: FirewallIPv6([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - }), - BS: BackendService("UDP"), - HC: HealthCheck(), - HCFirewallIPv6: HealthCheckFirewallIPv6(), - } -} - -// L3ResourcesIPv6 returns GCE resources for a mixed protocol IPv6 ILB, that listens on tcp:80, tcp:443 and udp:53 -func L3ResourcesIPv6() GCEResources { - return GCEResources{ - FwdRuleIPv6: ForwardingRuleL3IPv6(), - FirewallIPv6: FirewallIPv6([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - BS: BackendService("UNSPECIFIED"), - HC: HealthCheck(), - HCFirewallIPv6: HealthCheckFirewallIPv6(), - } -} - -// TCPResourcesDualStack returns GCE resources for a TCP dual stack ILB that listens on ports 80 and 443 -func TCPResourcesDualStack() GCEResources { - return GCEResources{ - FwdRuleIPv4: ForwardingRuleTCP([]string{"80", "443"}), - FirewallIPv4: Firewall([]*compute.FirewallAllowed{ - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - FwdRuleIPv6: ForwardingrukeRuleTCPIPv6([]string{"80", "443"}), - FirewallIPv6: FirewallIPv6([]*compute.FirewallAllowed{ - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - BS: BackendService("TCP"), - HC: HealthCheck(), - HCFirewallIPv4: HealthCheckFirewall(), - HCFirewallIPv6: HealthCheckFirewallIPv6(), - } -} - -// UDPResourcesDualStack returns GCE resources for an UDP dual stack ILB that listens on port 53 -func UDPResourcesDualStack() GCEResources { - return GCEResources{ - FwdRuleIPv4: ForwardingRuleUDP([]string{"53"}), - FirewallIPv4: Firewall([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - }), - FwdRuleIPv6: ForwardingRuleUDPIPv6([]string{"53"}), - FirewallIPv6: FirewallIPv6([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - }), - BS: BackendService("UDP"), - HC: HealthCheck(), - HCFirewallIPv4: HealthCheckFirewall(), - HCFirewallIPv6: HealthCheckFirewallIPv6(), - } -} - -// L3ResourcesDualStack returns GCE resources for a mixed protocol dual stack ILB, that listens on tcp:80, tcp:443 and udp:53 -func L3ResourcesDualStack() GCEResources { - return GCEResources{ - FwdRuleIPv4: ForwardingRuleL3(), - FirewallIPv4: Firewall([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - FwdRuleIPv6: ForwardingRuleL3IPv6(), - FirewallIPv6: FirewallIPv6([]*compute.FirewallAllowed{ - {IPProtocol: "udp", Ports: []string{"53"}}, - {IPProtocol: "tcp", Ports: []string{"80", "443"}}, - }), - BS: BackendService("UNSPECIFIED"), - HC: HealthCheck(), - HCFirewallIPv4: HealthCheckFirewall(), - HCFirewallIPv6: HealthCheckFirewallIPv6(), - } -} +const ( + ForwardingRuleTCPIPv4Name = "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i" + ForwardingRuleTCPIPv6Name = "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6" + ForwardingRuleUDPIPv4Name = "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i" + ForwardingRuleUDPIPv6Name = "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6" + ForwardingRuleL3IPv4Name = "k8s2-l3-axyqjz2d-test-namespace-test-name-yuvhdy7i" + ForwardingRuleL3IPv6Name = "k8s2-l3-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6" + ForwardingRuleLegacyName = "a" + TestUID + ForwardingRuleLegacyIPv6Name = "a" + TestUID + "-ipv6" + BackendServiceName = "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i" + FirewallIPv4Name = "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i" + FirewallIPv6Name = "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6" + HealthCheckFirewallIPv4Name = "k8s2-axyqjz2d-l4-shared-hc-fw" + HealthCheckFirewallIPv6Name = "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6" + HealthCheckName = "k8s2-axyqjz2d-l4-shared-hc" +) -// HealthCheck returns shared HealthCheck -func HealthCheck() *compute.HealthCheck { - return &compute.HealthCheck{ - Name: "k8s2-axyqjz2d-l4-shared-hc", - CheckIntervalSec: 8, - TimeoutSec: 1, - HealthyThreshold: 1, - UnhealthyThreshold: 3, - Type: "HTTP", - HttpHealthCheck: &compute.HTTPHealthCheck{Port: 10256, RequestPath: "/healthz"}, - Description: `{"networking.gke.io/service-name":"","networking.gke.io/api-version":"ga","networking.gke.io/resource-description":"This resource is shared by all L4 ILB Services using ExternalTrafficPolicy: Cluster."}`, +// GCEResources collects all GCE resources that are managed by the mixed protocol LBs +type GCEResources struct { + ForwardingRules map[string]*compute.ForwardingRule + Firewalls map[string]*compute.Firewall + HealthCheck *composite.HealthCheck + BackendService *compute.BackendService +} + +// Create creates resources in the specified cloud +func (r GCEResources) Create(cloud *gce.Cloud) error { + if r.BackendService != nil { + if err := cloud.CreateRegionBackendService(r.BackendService, cloud.Region()); err != nil { + return fmt.Errorf("fakeGCE.CreateBackendService() returned error %w", err) + } + } + + if r.HealthCheck != nil { + provider := healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA, klog.TODO()) + if err := provider.Create(r.HealthCheck); err != nil { + return fmt.Errorf("healthCheckProvider.Create() returned error %w", err) + } + } + + for _, fw := range r.Firewalls { + if err := cloud.CreateFirewall(fw); err != nil { + return fmt.Errorf("fakeGCE.CreateFirewall() returned error %w", err) + } + } + + for _, fr := range r.ForwardingRules { + if err := cloud.CreateRegionForwardingRule(fr, cloud.Region()); err != nil { + return fmt.Errorf("fakeGCE.CreateForwardingRule() returned error %w", err) + } + } + + return nil +} + +// VerifyResourcesExist checks if the resources are in the cloud. +// We use testing.T to return multiple nicely formatted errors. +func VerifyResourcesExist(t *testing.T, cloud *gce.Cloud, want GCEResources) { + t.Helper() + + for frName, wantFr := range want.ForwardingRules { + fr, err := cloud.GetRegionForwardingRule(frName, wantFr.Region) + if err != nil { + t.Errorf("cloud.GetForwardingRule(%v) returned error %v", frName, err) + } + ignoreFields := cmpopts.IgnoreFields(compute.ForwardingRule{}, "SelfLink") + if diff := cmp.Diff(wantFr, fr, ignoreFields); diff != "" { + t.Errorf("Forwarding rule mismatch (-want +got):\n%s", diff) + } + } + + for fwName, wantFw := range want.Firewalls { + fw, err := cloud.GetFirewall(fwName) + if err != nil { + t.Errorf("cloud.GetFirewall(%v) returned error %v", fwName, err) + } + ignoreFields := cmpopts.IgnoreFields(compute.Firewall{}, "SelfLink") + sortSourceRanges := cmpopts.SortSlices(func(x, y string) bool { return x < y }) + if diff := cmp.Diff(wantFw, fw, ignoreFields, sortSourceRanges); diff != "" { + t.Errorf("Firewall mismatch (-want +got):\n%s", diff) + } + } + + if want.HealthCheck != nil { + provider := healthchecksprovider.NewHealthChecks(cloud, meta.VersionGA, klog.TODO()) + hc, err := provider.Get(want.HealthCheck.Name, want.HealthCheck.Scope) + if err != nil { + t.Errorf("healthCheckProvider.Get(%v) got err: %v", want.HealthCheck.Name, err) + } + ignoreFields := cmpopts.IgnoreFields(composite.HealthCheck{}, "SelfLink", "Scope") + if diff := cmp.Diff(want.HealthCheck, hc, ignoreFields); diff != "" { + t.Errorf("Health check mismatch (-want +got):\n%s", diff) + } + } + + if want.BackendService != nil { + bs, err := cloud.GetRegionBackendService(want.BackendService.Name, want.BackendService.Region) + if err != nil { + t.Errorf("cloud.GetBackendService() returned error %v", err) + } + ignoreFields := cmpopts.IgnoreFields(compute.BackendService{}, "SelfLink", "ConnectionDraining", "Region") + if diff := cmp.Diff(want.BackendService, bs, ignoreFields); diff != "" { + t.Errorf("Backend service mismatch (-want +got):\n%s", diff) + } + } +} + +// VerifyResourcesCleanedUp checks if the old resources are removed from the cloud. +// We use testing.T to return multiple nicely formatted errors. +// We don't need to clean up firewall for health checks, since they are shared +func VerifyResourcesCleanedUp(t *testing.T, cloud *gce.Cloud, old, want GCEResources) { + t.Helper() + + for frName := range old.ForwardingRules { + if _, ok := want.ForwardingRules[frName]; ok { + continue + } + fr, err := cloud.GetRegionForwardingRule(frName, cloud.Region()) + if utils.IgnoreHTTPNotFound(err) != nil { + t.Errorf("cloud.GetForwardingRule() returned error %v", err) + } + if fr != nil { + t.Errorf("found forwarding rule %v, which should have been deleted", fr.Name) + } + } + + for fwName := range old.Firewalls { + isShared := fwName == HealthCheckFirewallIPv4Name || fwName == HealthCheckFirewallIPv6Name + if isShared { + continue + } + if _, ok := want.Firewalls[fwName]; ok { + continue + } + fw, err := cloud.GetFirewall(fwName) + if utils.IgnoreHTTPNotFound(err) != nil { + t.Errorf("cloud.GetFirewall() returned error %v", err) + } + if fw != nil { + t.Errorf("found firewall %v, which should have been deleted", fw.Name) + } + } + + if old.HealthCheck != nil && (want.HealthCheck == nil || want.HealthCheck.Name != old.HealthCheck.Name) { + hc, err := cloud.GetHealthCheck(old.HealthCheck.Name) + if utils.IgnoreHTTPNotFound(err) != nil { + t.Errorf("cloud.GetHealthCheck() returned error %v", err) + } + if hc != nil { + t.Errorf("found health check %v, which should have been deleted", hc.Name) + } + } + + if old.BackendService != nil && (want.BackendService == nil || want.BackendService.Name != old.BackendService.Name) { + bs, err := cloud.GetRegionBackendService(old.BackendService.Name, old.BackendService.Region) + if utils.IgnoreHTTPNotFound(err) != nil { + t.Errorf("cloud.GetBackendService() returned error %v", err) + } + if bs != nil { + t.Errorf("found backend service %v, which should have been deleted", bs.Name) + } } } // HealthCheckFirewall returns Firewall for HealthCheck func HealthCheckFirewall() *compute.Firewall { return &compute.Firewall{ - Name: "k8s2-axyqjz2d-l4-shared-hc-fw", + Name: HealthCheckFirewallIPv4Name, Allowed: []*compute.FirewallAllowed{{IPProtocol: "TCP", Ports: []string{"10256"}}}, // GCE defined health check ranges SourceRanges: []string{"130.211.0.0/22", "35.191.0.0/16", "209.85.152.0/22", "209.85.204.0/22"}, @@ -181,7 +195,7 @@ func HealthCheckFirewall() *compute.Firewall { // HealthCheckFirewallIPv6 returns Firewall for HealthCheck func HealthCheckFirewallIPv6() *compute.Firewall { return &compute.Firewall{ - Name: "k8s2-axyqjz2d-l4-shared-hc-fw-ipv6", + Name: HealthCheckFirewallIPv6Name, Allowed: []*compute.FirewallAllowed{{IPProtocol: "TCP", Ports: []string{"10256"}}}, // GCE defined health check ranges SourceRanges: []string{"2600:2d00:1:b029::/64"}, @@ -190,114 +204,10 @@ func HealthCheckFirewallIPv6() *compute.Firewall { } } -// ForwardingRuleL3 returns an L3 Forwarding Rule for ILB -func ForwardingRuleL3() *compute.ForwardingRule { - return &compute.ForwardingRule{ - Name: "k8s2-l3-axyqjz2d-test-namespace-test-name-yuvhdy7i", - Region: "us-central1", - IPProtocol: "L3_DEFAULT", - AllPorts: true, - BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - LoadBalancingScheme: "INTERNAL", - NetworkTier: "PREMIUM", - Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, - } -} - -// ForwardingRuleL3IPv6 returns an L3 Forwarding Rule for ILB -func ForwardingRuleL3IPv6() *compute.ForwardingRule { - return &compute.ForwardingRule{ - Name: "k8s2-l3-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", - Region: "us-central1", - IPProtocol: "L3_DEFAULT", - IpVersion: "IPV6", - AllPorts: true, - BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - LoadBalancingScheme: "INTERNAL", - NetworkTier: "PREMIUM", - Description: `{"networking.gke.io/service-name":"test-namespace/test-name"}`, - } -} - -// ForwardingRuleUDP returns a UDP Forwarding Rule with specified ports -func ForwardingRuleUDP(ports []string) *compute.ForwardingRule { - return &compute.ForwardingRule{ - Name: "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i", - Region: "us-central1", - IPProtocol: "UDP", - Ports: ports, - BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - LoadBalancingScheme: "INTERNAL", - NetworkTier: "PREMIUM", - Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, - } -} - -// ForwardingRuleUDPIPv6 returns a UDP Forwarding Rule with specified ports -func ForwardingRuleUDPIPv6(ports []string) *compute.ForwardingRule { - return &compute.ForwardingRule{ - Name: "k8s2-udp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", - Region: "us-central1", - IPProtocol: "UDP", - IpVersion: "IPV6", - Ports: ports, - BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - LoadBalancingScheme: "INTERNAL", - NetworkTier: "PREMIUM", - Description: `{"networking.gke.io/service-name":"test-namespace/test-name"}`, - } -} - -// ForwardingRuleTCP returns a TCP Forwarding Rule with specified ports -func ForwardingRuleTCP(ports []string) *compute.ForwardingRule { - return &compute.ForwardingRule{ - Name: "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i", - Region: "us-central1", - IPProtocol: "TCP", - Ports: ports, - BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - LoadBalancingScheme: "INTERNAL", - NetworkTier: "PREMIUM", - Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, - } -} - -// ForwardingrukeRuleTCPIPv6 returns a TCP Forwarding Rule with specified ports -func ForwardingrukeRuleTCPIPv6(ports []string) *compute.ForwardingRule { - return &compute.ForwardingRule{ - Name: "k8s2-tcp-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", - Region: "us-central1", - IPProtocol: "TCP", - IpVersion: "IPV6", - Ports: ports, - BackendService: "https://www.googleapis.com/compute/v1/projects/test-project/regions/us-central1/backendServices/k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - LoadBalancingScheme: "INTERNAL", - NetworkTier: "PREMIUM", - Description: `{"networking.gke.io/service-name":"test-namespace/test-name"}`, - } -} - -// BackendService returns Backend Service for ILB -// protocol should be set to: -// - `UNSPECIFIED` for mixed protocol (L3) -// - `TCP` for TCP only -// - `UDP` for UDP only -func BackendService(protocol string) *compute.BackendService { - return &compute.BackendService{ - Name: "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", - Region: "us-central1", - Protocol: protocol, - SessionAffinity: "NONE", - LoadBalancingScheme: "INTERNAL", - HealthChecks: []string{"https://www.googleapis.com/compute/v1/projects/test-project/global/healthChecks/k8s2-axyqjz2d-l4-shared-hc"}, - Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, - } -} - // Firewall returns ILB Firewall with specified allowed rules func Firewall(allowed []*compute.FirewallAllowed) *compute.Firewall { return &compute.Firewall{ - Name: "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i", + Name: FirewallIPv4Name, Allowed: allowed, Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, SourceRanges: []string{"0.0.0.0/0"}, @@ -308,7 +218,7 @@ func Firewall(allowed []*compute.FirewallAllowed) *compute.Firewall { // FirewallIPv6 returns ILB Firewall with specified allowed rules func FirewallIPv6(allowed []*compute.FirewallAllowed) *compute.Firewall { return &compute.Firewall{ - Name: "k8s2-axyqjz2d-test-namespace-test-name-yuvhdy7i-ipv6", + Name: FirewallIPv6Name, Allowed: allowed, Description: `{"networking.gke.io/service-name":"test-namespace/test-name","networking.gke.io/api-version":"ga"}`, SourceRanges: []string{"0::0/0"}, diff --git a/pkg/loadbalancers/mixedprotocoltest/spec.go b/pkg/loadbalancers/mixedprotocoltest/spec.go index bca5f6a872..be6742b19d 100644 --- a/pkg/loadbalancers/mixedprotocoltest/spec.go +++ b/pkg/loadbalancers/mixedprotocoltest/spec.go @@ -7,8 +7,9 @@ import ( // SpecIPv4 returns a specification for a IPv6 LB with specified tcp and udp ports func SpecIPv4(tcpPorts []int32, udpPorts []int32) api_v1.ServiceSpec { return api_v1.ServiceSpec{ - Type: api_v1.ServiceTypeLoadBalancer, - Ports: getPorts(tcpPorts, udpPorts), + Type: api_v1.ServiceTypeLoadBalancer, + Ports: getPorts(tcpPorts, udpPorts), + SessionAffinity: api_v1.ServiceAffinityNone, } } @@ -17,10 +18,11 @@ func SpecIPv6(tcpPorts []int32, udpPorts []int32) api_v1.ServiceSpec { policy := api_v1.IPFamilyPolicySingleStack return api_v1.ServiceSpec{ - IPFamilies: []api_v1.IPFamily{"IPv6"}, - IPFamilyPolicy: &policy, - Type: api_v1.ServiceTypeLoadBalancer, - Ports: getPorts(tcpPorts, udpPorts), + IPFamilies: []api_v1.IPFamily{"IPv6"}, + IPFamilyPolicy: &policy, + Type: api_v1.ServiceTypeLoadBalancer, + Ports: getPorts(tcpPorts, udpPorts), + SessionAffinity: api_v1.ServiceAffinityNone, } } @@ -29,10 +31,11 @@ func SpecDualStack(tcpPorts []int32, udpPorts []int32) api_v1.ServiceSpec { policy := api_v1.IPFamilyPolicyRequireDualStack return api_v1.ServiceSpec{ - IPFamilies: []api_v1.IPFamily{"IPv4", "IPv6"}, - IPFamilyPolicy: &policy, - Type: api_v1.ServiceTypeLoadBalancer, - Ports: getPorts(tcpPorts, udpPorts), + IPFamilies: []api_v1.IPFamily{"IPv4", "IPv6"}, + IPFamilyPolicy: &policy, + Type: api_v1.ServiceTypeLoadBalancer, + Ports: getPorts(tcpPorts, udpPorts), + SessionAffinity: api_v1.ServiceAffinityNone, } }