SecureComms: E2e test with KBS

Install KBS and test SecureComms with KBS
Based on confidential-containers#2072 which should be merged first

Signed-off-by: David Hadas <[email protected]>

src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go

@@ -34,7 +34,9 @@ var logger = log.New(log.Writer(), "[forwarder] ", log.LstdFlags|log.Lmsgprefix)
 type Config struct {
 	tlsConfig           *tlsutil.TLSConfig
 	daemonConfig        daemon.Config
-	configPath          string
+	apfConfig           daemon.ApfConfig
+	daemonConfigPath    string
+	apfConfigPath       string
 	listenAddr          string
 	kataAgentSocketPath string
 	podNamespace        string
@@ -52,6 +54,8 @@ func load(path string, obj interface{}) error {
 		return fmt.Errorf("failed to decode a Agent Protocol Forwarder config file file: %s: %w", path, err)
 	}
 
+	logger.Printf("succesful loading config from %s\n", path)
+
 	return nil
 }
 
@@ -68,7 +72,8 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 
 	cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) {
 		flags.BoolVar(&showVersion, "version", false, "Show version")
-		flags.StringVar(&cfg.configPath, "config", daemon.DefaultConfigPath, "Path to a daemon config file")
+		flags.StringVar(&cfg.daemonConfigPath, "config", daemon.DefaultDaemonConfigPath, "Path to a daemon config file")
+		flags.StringVar(&cfg.apfConfigPath, "apf-config", daemon.DefaultAPFConfigPath, "Path to APF config file")
 		flags.StringVar(&cfg.listenAddr, "listen", daemon.DefaultListenAddr, "Listen address")
 		flags.StringVar(&cfg.kataAgentSocketPath, "kata-agent-socket", daemon.DefaultKataAgentSocketPath, "Path to a kata agent socket")
 		flags.StringVar(&cfg.podNamespace, "pod-namespace", daemon.DefaultPodNamespace, "Path to the network namespace where the pod runs")
@@ -89,13 +94,20 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 		cmd.Exit(0)
 	}
 
-	if err := load(cfg.configPath, &cfg.daemonConfig); err != nil {
+	if err := load(cfg.daemonConfigPath, &cfg.daemonConfig); err != nil {
+		return nil, err
+	}
+
+	if err := load(cfg.apfConfigPath, &cfg.apfConfig); err != nil {
 		return nil, err
 	}
 
+	if cfg.apfConfig.SecureComms {
+		secureComms = true
+	}
+
 	if secureComms || cfg.daemonConfig.SecureComms {
 		var inbounds, outbounds []string
-
 		ppssh.Singleton()
 		host, port, err := net.SplitHostPort(cfg.listenAddr)
 		if err != nil {

src/cloud-api-adaptor/docs/SecureComms.md

@@ -122,7 +122,13 @@ kubectl -n trustee-operator-system exec deployment/trustee-deployment --containe
 kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
 ```
 
-### Build a podvm that enforces Secure-Comms
+### Build a podvm that enforces Secure-Comms (Optional)
+
+This stage is optional, it should be used in following two cases:
+1. Constructing a podvm that enforces Secure Comms to always be activated
+2. Configuring the podvm for Secure-Comms Inbounds and outbounds 
+
+An alternative to this stage is to use InitData to enforce Secure Comms to be activated. From security standpoint, this alternative is helpful in production when InitData is measured and attested together with the podvm measurement.
 
 Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include:
 ```sh
@@ -170,6 +176,7 @@ url = 'http://127.0.0.1:8080'
 EOF
 export INITDATA=`base64 -w 0 /tmp/initdata.txt`
 kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed 's/^\s*INITDATA: .*/  INITDATA: '$INITDATA'/'|kubectl apply -f -
+
 ```
 
 You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)

src/cloud-api-adaptor/pkg/forwarder/forwarder.go

@@ -27,13 +27,15 @@ const (
 	DefaultListenHost          = "0.0.0.0"
 	DefaultListenPort          = "15150"
 	DefaultListenAddr          = DefaultListenHost + ":" + DefaultListenPort
-	DefaultConfigPath          = "/run/peerpod/daemon.json"
+	DefaultDaemonConfigPath    = "/run/peerpod/daemon.json"
+	DefaultAPFConfigPath       = "/run/peerpod/apf.json"
 	DefaultPodNetworkSpecPath  = "/run/peerpod/podnetwork.json"
 	DefaultKataAgentSocketPath = "/run/kata-containers/agent.sock"
 	DefaultPodNamespace        = "/run/netns/podns"
 	AgentURLPath               = "/agent"
 )
 
+// Unmeasured Configuration via User Data
 type Config struct {
 	PodNetwork   *tunneler.Config `json:"pod-network"`
 	PodNamespace string           `json:"pod-namespace"`
@@ -50,6 +52,11 @@ type Config struct {
 	SecureComms          bool   `json:"sc,omitempty"`
 }
 
+// Measured Configuration via InitData
+type ApfConfig struct {
+	SecureComms bool `json:"sc,omitempty"`
+}
+
 type Daemon interface {
 	Start(ctx context.Context) error
 	Shutdown() error

src/cloud-api-adaptor/pkg/userdata/provision.go

@@ -17,6 +17,7 @@ import (
 	toml "github.com/pelletier/go-toml/v2"
 	"gopkg.in/yaml.v2"
 
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/initdata"
 	. "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/paths"
 )
@@ -38,7 +39,7 @@ const (
 
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
 var WriteFilesList = []string{AACfgPath, CDHCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath}
-var InitdDataFilesList = []string{AACfgPath, CDHCfgPath, PolicyPath}
+var InitdDataFilesList = []string{AACfgPath, CDHCfgPath, PolicyPath, forwarder.DefaultAPFConfigPath}
 
 type Config struct {
 	fetchTimeout  int

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -119,6 +119,16 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
 		initdata = properties["INITDATA"]
 	}
 
+	secure_comms := "false"
+	if properties["SECURE_COMMS"] != "" {
+		secure_comms = properties["SECURE_COMMS"]
+	}
+
+	secure_comms_kbs_addr := ""
+	if properties["SECURE_COMMS_KBS_ADDR"] != "" {
+		secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"]
+	}
+
 	// TODO: Check network and storage are not nil?
 	return &LibvirtProvisioner{
 		conn:                    conn,
@@ -373,6 +383,7 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	for k, v := range mapProps {
 		if properties[k] != v[0] {
+			fmt.Printf("TESTONLY setting  %s: %s\n", v[1], properties[k])
 			if err = lio.Overlay.SetKustomizeConfigMapGeneratorLiteral("peer-pods-cm",
 				v[1], properties[k]); err != nil {
 				return err

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -56,6 +56,7 @@ type CloudAPIAdaptor struct {
 	installOverlay       InstallOverlay       // Pointer to the kustomize overlay
 	runtimeClass         *nodev1.RuntimeClass // The Kata Containers runtimeclass
 	rootSrcDir           string               // The root src directory of cloud-api-adaptor
+	finish               chan bool            // The test was finished
 }
 
 type NewInstallOverlayFunc func(installDir, provider string) (InstallOverlay, error)
@@ -105,6 +106,7 @@ func NewCloudAPIAdaptor(provider string, installDir string) (*CloudAPIAdaptor, e
 		installOverlay:       overlay,
 		runtimeClass:         &nodev1.RuntimeClass{ObjectMeta: metav1.ObjectMeta{Name: "kata-remote", Namespace: ""}},
 		rootSrcDir:           filepath.Dir(installDir),
+		finish:               make(chan bool),
 	}, nil
 }
 
@@ -221,6 +223,7 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error
 		return err
 	}
 
+	close(p.finish)
 	return nil
 }