Skip to content

Commit

Permalink
Support multiple DB, push hydrophone logs to test output
Browse files Browse the repository at this point in the history 
Signed-off-by: Derek Nola <[email protected]>
  • Loading branch information
@dereknola
dereknola committed Jan 31, 2025
1 parent f0100d0 commit 82b7fb2
Show file tree
Hide file tree
Showing 2 changed files with 119 additions and 21 deletions.
63 changes: 53 additions & 10 deletions tests/docker/conformance/conformance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,20 @@ import (
"flag"
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"time"

tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var k3sImage = flag.String("k3sImage", "", "The k3s image used to provision containers")
var db = flag.String("db", "", "The database to use for the tests (sqlite, etcd, mysql, postgres)")
var serial = flag.Bool("serial", false, "Run the Serial Conformance Tests")
var config *tester.TestConfig

Expand All @@ -30,6 +34,7 @@ var _ = Describe("Conformance Tests", Ordered, func() {
var err error
config, err = tester.NewTestConfig(*k3sImage)
Expect(err).NotTo(HaveOccurred())
config.DBType = *db
Expect(config.ProvisionServers(1)).To(Succeed())
Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func() error {
Expand Down Expand Up @@ -59,30 +64,56 @@ var _ = Describe("Conformance Tests", Ordered, func() {
if *serial {
Skip("Skipping parallel conformance tests")
}
cmd := fmt.Sprintf("%s --focus=\"Conformance\" --skip=\"Serial|Flaky\" -p %d --extra-ginkgo-args=\"%s\" --kubeconfig %s",
cmd := fmt.Sprintf("%s --focus=\"Conformance\" --skip=\"Serial|Flaky\" -v 2 -p %d --kubeconfig %s",
filepath.Join(config.TestDir, "hydrophone"),
runtime.NumCPU()/2,
"--poll-progress-after=60s,--poll-progress-interval=30s",
config.KubeconfigFile)
By("Hydrophone: " + cmd)
res, err := tester.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), res)
hc, err := StartCmd(cmd)
Expect(err).NotTo(HaveOccurred())
// Periodically check the number of tests that have run, since the hydrophone output does not support a progress status
// Taken from https://github.com/kubernetes-sigs/hydrophone/issues/223#issuecomment-2547174722
go func() {
cmd := fmt.Sprintf("kubectl exec -n=conformance e2e-conformance-test -c output-container --kubeconfig=%s -- cat /tmp/results/e2e.log | grep -o \"•\" | wc -l",
config.KubeconfigFile)
for i := 1; ; i++ {
time.Sleep(120 * time.Second)
if hc.ProcessState != nil {
break
}
res, _ := tester.RunCommand(cmd)
res = strings.TrimSpace(res)
fmt.Printf("Status Report %d: %s tests complete\n", i, res)
}
}()
Expect(hc.Wait()).To(Succeed())
})
It("should run serial conformance tests", func() {
if !*serial {
Skip("Skipping serial conformance tests")
}
cmd := fmt.Sprintf("%s -o %s --focus=\"Serial\" --skip=\"Flaky\" --extra-ginkgo-args=\"%s\" --kubeconfig %s",
cmd := fmt.Sprintf("%s --focus=\"Serial\" --skip=\"Flaky\" -v 2 --kubeconfig %s",
filepath.Join(config.TestDir, "hydrophone"),
filepath.Join(config.TestDir, "logs"),
"--poll-progress-after=60s,--poll-progress-interval=30s",
config.KubeconfigFile)
By("Hydrophone: " + cmd)
res, err := tester.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), res)
hc, err := StartCmd(cmd)
Expect(err).NotTo(HaveOccurred())
go func() {
cmd := fmt.Sprintf("kubectl exec -n=conformance e2e-conformance-test -c output-container --kubeconfig=%s -- cat /tmp/results/e2e.log | grep -o \"•\" | wc -l",
config.KubeconfigFile)
for i := 1; ; i++ {
time.Sleep(120 * time.Second)
if hc.ProcessState != nil {
break
}
res, _ := tester.RunCommand(cmd)
res = strings.TrimSpace(res)
fmt.Printf("Status Report %d: %s tests complete\n", i, res)
}
}()
Expect(hc.Wait()).To(Succeed())
})
})

})

var failed bool
Expand All @@ -95,3 +126,15 @@ var _ = AfterSuite(func() {
config.Cleanup()
}
})

// StartCmd starts a command and pipes its output to
// the ginkgo Writr, with the expectation to poll the progress of the command
func StartCmd(cmd string) (*exec.Cmd, error) {
c := exec.Command("sh", "-c", cmd)
c.Stdout = GinkgoWriter
c.Stderr = GinkgoWriter
if err := c.Start(); err != nil {
return c, err
}
return c, nil
}
77 changes: 66 additions & 11 deletions tests/docker/test-helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type TestConfig struct {
KubeconfigFile string
Token string
K3sImage string
DBType string
Servers []Server
Agents []DockerNode
ServerYaml string
Expand Down Expand Up @@ -121,16 +122,23 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
yamlMount = fmt.Sprintf("--mount type=bind,src=%s,dst=/etc/rancher/k3s/config.yaml", filepath.Join(config.TestDir, fmt.Sprintf("server-%d.yaml", i)))
}

var joinOrStart string
if numOfServers > 0 {
if i == 0 {
joinOrStart = "--cluster-init"
} else {
if config.Servers[0].URL == "" {
return fmt.Errorf("first server URL is empty")
}
joinOrStart = fmt.Sprintf("--server %s", config.Servers[0].URL)
var joinServer string
var dbConnect string
var err error
if i == 0 {
dbConnect, err = config.setupDatabase(true)
if err != nil {
return err
}
} else {
dbConnect, err = config.setupDatabase(false)
if err != nil {
return err
}
if config.Servers[0].URL == "" {
return fmt.Errorf("first server URL is empty")
}
joinServer = fmt.Sprintf("--server %s", config.Servers[0].URL)
}
newServer := Server{
DockerNode: DockerNode{
Expand Down Expand Up @@ -169,7 +177,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
}
// The pipe requires that we use sh -c with "" to run the command
cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
joinOrStart+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)))
if out, err := newServer.RunCmdOnNode(cmd); err != nil {
return fmt.Errorf("failed to start server: %s: %v", out, err)
}
Expand All @@ -189,7 +197,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
os.Getenv("REGISTRY_CLUSTER_ARGS"),
yamlMount,
config.K3sImage,
"server", joinOrStart, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
"server", dbConnect, joinServer, os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i))}, " ")
if out, err := RunCommand(dRun); err != nil {
return fmt.Errorf("failed to run server container: %s: %v", out, err)
}
Expand Down Expand Up @@ -220,6 +228,41 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
return copyAndModifyKubeconfig(config)
}

// setupDatabase will start the configured database if startDB is true,
// and return the correct flag to join the configured database
func (config *TestConfig) setupDatabase(startDB bool) (string, error) {

joinFlag := ""
startCmd := ""
switch config.DBType {
case "mysql":
startCmd = "docker run -d --name mysql -e MYSQL_ROOT_PASSWORD=docker -p 3306:3306 mysql:8.4"
joinFlag = "--datastore-endpoint='mysql://root:docker@tcp(172.17.0.1:3306)/k3s'"
case "postgres":
startCmd = "docker run -d --name postgres -e POSTGRES_PASSWORD=docker -p 5432:5432 postgres:16-alpine"
joinFlag = "--datastore-endpoint='postgres://postgres:docker@tcp(172.17.0.1:5432)/k3s'"
case "etcd":
if startDB {
joinFlag = "--cluster-init"
}
case "sqlite":
fallthrough
default:
config.DBType = "sqlite"
return "", nil
}

if startDB {
if out, err := RunCommand(startCmd); err != nil {
return "", fmt.Errorf("failed to start %s container: %s: %v", config.DBType, out, err)
}
// Wait for DB to start
time.Sleep(10 * time.Second)
}
return joinFlag, nil

}

func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
if err := checkVersionSkew(config); err != nil {
return err
Expand Down Expand Up @@ -346,6 +389,18 @@ func (config *TestConfig) Cleanup() error {
}
}

// Stop DB if it was started
if config.DBType != "etcd" {
cmd := fmt.Sprintf("docker stop %s", config.DBType)
if _, err := RunCommand(cmd); err != nil {
errs = append(errs, fmt.Errorf("failed to stop %s: %v", config.DBType, err))
}
cmd = fmt.Sprintf("docker rm %s", config.DBType)
if _, err := RunCommand(cmd); err != nil {
errs = append(errs, fmt.Errorf("failed to remove %s: %v", config.DBType, err))
}
}

// Error out if we hit any issues
if len(errs) > 0 {
return fmt.Errorf("cleanup failed: %v", errs)
Expand Down

0 comments on commit 82b7fb2

Please sign in to comment.