podvm: Allow specifying pod subnet CIDRs explicitly

This is useful for setting explicit routes in the pod namespace
for the vxlan device

Signed-off-by: Pradipta Banerjee <[email protected]>

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -123,6 +123,8 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.IntVar(&cfg.networkConfig.VXLAN.Port, "vxlan-port", vxlan.DefaultVXLANPort, "VXLAN UDP port number (VXLAN tunnel mode only")
 		flags.IntVar(&cfg.networkConfig.VXLAN.MinID, "vxlan-min-id", vxlan.DefaultVXLANMinID, "Minimum VXLAN ID (VXLAN tunnel mode only")
 		flags.BoolVar(&cfg.networkConfig.ExternalNetViaPodVM, "ext-network-via-podvm", false, "Enable external networking via pod VM")
+		// Local pod subnets. This will be used by APF to create routes for local pod subnets when using external networking via pod VM
+		flags.Var(&cfg.networkConfig.PodSubnetCIDRs, "pod-subnet-cidrs", "Local pod subnets")
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
 		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")

src/cloud-api-adaptor/docs/external-network.md

@@ -0,0 +1,35 @@
+# Introduction
+
+Currently, all pod traffic is routed via the worker node (WN).
+This is by design and is  fine as all existing network policies that were applicable to pods running on the WN also apply to peer pods.
+However, the peer pods approach has enabled some additional use cases that were earlier impossible or impractical. Concurrently, these new use cases have resulted in new requirements.
+
+Consider a scenario where you are running a Kind K8s cluster on your laptop, experimenting with an AI model, and needing additional computing power or GPUs. With peer pods, you can spin up a pod using the appropriate instance type from the Kind K8s cluster running on your laptop.
+
+From a networking standpoint, this means that there should be bidirectional connectivity between the cloud and your development setup.
+
+One solution is to use a VPN, which complicates the setup. A relatively easier alternative is to use a public IP associated with the cloud instance for control-plane traffic and use the cloud instance network for external connectivity requirements (like accessing cloud object storage for data) instead of routing all traffic via your development setup.
+
+The external network connectivity for the pod via pod VM network is enabled via the
+option `ext-network-via-podvm` in cloud-api-adaptor. The equivalent option in the `peer-pods-cm` is `EXTERNAL_NETWORK_VIA_PODVM`
+
+The prerequisite is for the pod VM to have a secondary interface with an IP. This interface will be moved to the pod network namespace and default routes adjusted so that pod network traverses via worker node, and any other traffic uses the secondary interface.
+
+## Specifying Pod subnet CIDRs
+
+When using the `ext-network-via-podvm` feature, the default route for the pod is set to the secondary interface. Traffic on the pod subnet is routed via the pod network specific interface (vxlan). The pod subnet is auto detected based on the pod IP by Linux for the route. This may not be sufficient for all cases and you may want to specify pod subnets explicitly.
+We have an option for the same named `pod-subnet-cidrs` to provide a comma separated list of CIDRs that will be routed via the pod network specific interface.
+
+The pod subnet CIDR, service CIDR depends on Kubernetes CNI configuration. Here are some examples:
+
+* A generic way to detect the cluster service CIDR as mentioned [here](Ref: https://stackoverflow.com/questions/44190607/how-do-you-find-the-cluster-service-cidr-of-a-kubernetes-cluster)
+
+```sh
+SVCRANGE=$(echo '{"apiVersion":"v1","kind":"Service","metadata":{"name":"tst"},"spec":{"clusterIP":"1.1.1.1","ports":[{"port":443}]}}' | kubectl apply -f - 2>&1 | sed 's/.*valid IPs is //')
+echo $SVCRANGE
+```
+
+* The default pod CIDR for Flannel CNI: `10.244.0.0/16`
+* AWS VPC CNI uses VPC CIDR as the pod CIDR
+* Azure CNI uses the subnet CIDR as the pod CIDR
+* The default pod CIDR for OVN-Kubernetes: `10.128.0.0/14`

src/cloud-api-adaptor/entrypoint.sh

@@ -31,7 +31,7 @@ optionals+=""
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
 [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
 [[ "${EXTERNAL_NETWORK_VIA_PODVM}" ]] && optionals+="-ext-network-via-podvm  "
-
+[[ "${POD_SUBNET_CIDRS}" ]] && optionals+="-pod-subnet-cidrs ${POD_SUBNET_CIDRS} "
 
 test_vars() {
     for i in "$@"; do

src/cloud-api-adaptor/install/overlays/aws/kustomization.yaml

@@ -34,6 +34,8 @@ configMapGenerator:
   #- AWS_SUBNET_ID="" # if not set retrieved from IMDS
   #- TAGS="" # Uncomment and add key1=value1,key2=value2 etc if you want to use specific tags for podvm
   #- USE_PUBLIC_IP="true" # Uncomment if you want to use public ip for podvm
+  #- EXTERNAL_NETWORK_VIA_PODVM="true" # Uncomment if you want to use podvm as external network
+  #- POD_SUBNET_CIDRS="10.244.0.0/16,10.96.0.0/12" # Uncomment and set if you want to use specific subnet cidrs for podvm. Comma separated. The default is for a kind cluster
   #- ROOT_VOLUME_SIZE="30" # Uncomment and set if you want to use a specific root volume size. Defaults to 30
   #- FORWARDER_PORT="" # Uncomment and set if you want to use a specific port for agent-protocol-forwarder. Defaults to 15150
   #- PEERPODS_LIMIT_PER_NODE="10" # Max number of peer pods that can be created per node. Default is 10

src/cloud-api-adaptor/pkg/podnetwork/common.go

@@ -220,7 +220,7 @@ func isInterfaceFilteredOut(ifName string) bool {
 // Function to move the network interface to a different network namespace
 // Also set the default route and address for the interface in the new namespace
 
-func moveInterfaceToNamespace(srcNs, dstNs netops.Namespace, iface string, addrCIDR netip.Prefix, route *netops.Route) error {
+func moveInterfaceToNamespace(srcNs, dstNs netops.Namespace, iface string, addrCIDR netip.Prefix, defRoute *netops.Route) error {
 
 	// Get the network interface object
 	link, err := srcNs.LinkFind(iface)
@@ -261,9 +261,9 @@ func moveInterfaceToNamespace(srcNs, dstNs netops.Namespace, iface string, addrC
 	}
 
 	// Set the default route for the network interface in the new namespace
-	err = dstNs.RouteAdd(route)
+	err = dstNs.RouteAdd(defRoute)
 	if err != nil {
-		return fmt.Errorf("failed to set route %v for link %q: %w", route, iface, err)
+		return fmt.Errorf("failed to set route %v for link %q: %w", defRoute, iface, err)
 	}
 
 	return nil

src/cloud-api-adaptor/pkg/podnetwork/tunneler/config.go

@@ -3,6 +3,8 @@
 
 package tunneler
 
+import "strings"
+
 type TunnelerConfigurator interface {
 	Tunneler
 	Configure(*NetworkConfig, *Config) error
@@ -13,9 +15,25 @@ type NetworkConfig struct {
 	HostInterface       string
 	VXLAN               VXLANConfig
 	ExternalNetViaPodVM bool
+	PodSubnetCIDRs      SubnetCIDRs
 }
 
 type VXLANConfig struct {
 	Port  int
 	MinID int
 }
+
+type SubnetCIDRs []string
+
+func (i *SubnetCIDRs) String() string {
+	return strings.Join(*i, ", ")
+}
+
+func (i *SubnetCIDRs) Set(value string) error {
+	parts := strings.Split(value, ",")
+
+	for _, part := range parts {
+		*i = append(*i, strings.TrimSpace(part))
+	}
+	return nil
+}

src/cloud-api-adaptor/pkg/podnetwork/workernode.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/netops"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 )
 
 const DefaultTunnelType = "vxlan"
@@ -187,6 +189,26 @@ func (n *workerNode) Inspect(nsPath string) (*tunneler.Config, error) {
 		config.Routes = append(config.Routes, r)
 	}
 
+	// Add route for the subnet CIDRs in the new namespace
+	if n.PodSubnetCIDRs != nil {
+		for _, cidr := range n.PodSubnetCIDRs {
+			prefix, err := netip.ParsePrefix(cidr)
+			if err != nil {
+				logger.Printf("failed to parse CIDR %q: %w", cidr, err)
+				continue
+			}
+			route := &tunneler.Route{
+				Dst: prefix,
+				// The gateway address is 0.0.0.0
+				GW:       netip.Addr{},
+				Dev:      podInterface,
+				Scope:    netops.RouteScope(netlink.SCOPE_LINK),
+				Protocol: netops.RouteProtocol(unix.RTPROT_KERNEL),
+			}
+			config.Routes = append(config.Routes, route)
+		}
+	}
+
 	for _, neighbor := range neighbors {
 		n := &tunneler.Neighbor{
 			IP:           neighbor.IP,