From 7c4e48c8d59138a344acd5d4f32399326e56b5ca Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:19:09 -0700 Subject: [PATCH] ci: Add Cilium to CNI | LSG Integration Pipeline (#2574) * ci: Kernel upgrade template * ci: lsg cni intergration template * chore: address comments * ci: add IPAM repo/ver control * ci: add cilium * chore: template style * fix: DAG logic * ci: finalize changes * ci: update cilium filepaths * ci: address comments --- .../cni/lsg/kernel-upgrade-template.yaml | 12 +- .../lsg/lsg-cni-intergration-template.yaml | 179 +++++++++++++----- .pipelines/cni/lsg/pipeline.yaml | 60 +++--- test/internal/kubernetes/utils_create.go | 10 +- 4 files changed, 179 insertions(+), 82 deletions(-) diff --git a/.pipelines/cni/lsg/kernel-upgrade-template.yaml b/.pipelines/cni/lsg/kernel-upgrade-template.yaml index e4adfafa6e..83d5c11b58 100644 --- a/.pipelines/cni/lsg/kernel-upgrade-template.yaml +++ b/.pipelines/cni/lsg/kernel-upgrade-template.yaml @@ -47,7 +47,8 @@ steps: privList=`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'` for pod in $privList; do echo "-- Add proposed repository --" - kubectl exec -i -n kube-system $pod -- bash -c "add-apt-repository ppa:canonical-kernel-team/${{ parameters.ProposedRepoVersion }} -y" + kubectl exec -i -n kube-system $pod -- bash -c "add-apt-repository ppa:canonical-kernel-team/$(ProposedRepoVersion) -y" + echo "-- Check apt-cache --" kubectl exec -i -n kube-system $pod -- bash -c "apt-cache madison $(kernelType)" @@ -58,12 +59,11 @@ steps: for pod in $privList; do flag1="" - flag2="" echo "-- Confirm Proposed Repository --" temp=`kubectl exec -i -n kube-system $pod -- bash -c "apt-cache madison $(kernelType)" | grep proposed | awk '{print $3}'` for t in $temp; do - if [[ $t =~ ${{ parameters.KernelVersion }} ]]; then - echo "$t - looking for ${{ parameters.KernelVersion }}" + if [[ $t =~ $(KernelVersion) ]]; then + echo "$t - looking for $(KernelVersion)" flag1="true" fi done @@ -126,8 +126,8 @@ steps: echo "-- Confirm Ubuntu Kernel --" kernelOut=`kubectl exec -i -n kube-system $pod -- bash -c "uname -r"` ver=(${kernelOut//-/ }) # Slices $kernelOut with '-' - echo "$pod | kernel version - $kernelOut, expected - ${{ parameters.KernelVersion }}" - if [[ ! ${{ parameters.KernelVersion }} =~ ${ver[1]} ]]; then + echo "$pod | kernel version - $kernelOut, expected - $(KernelVersion)" + if [[ ! $(KernelVersion) =~ ${ver[1]} ]]; then exit 1 fi done diff --git a/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml b/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml index 78c8faff59..b6444e4173 100644 --- a/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml +++ b/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml @@ -9,13 +9,6 @@ parameters: arch: "" osSKU: Ubuntu -# CNIv2 -# + Should be able to scale up/down the pods successfully certain number of times. -# + Node reboot scenarios should be covered. -# + The CNS state should be validated with that of CNI state. -# + Pods should have ips assigned and connectivity/datapath test should be present. -# + CNS restart and validates the state - stages: - stage: create_${{ parameters.name }} variables: @@ -35,6 +28,7 @@ stages: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) make -C ./hack/aks ${{ parameters.clusterType }} \ AZCLI=az REGION=$(LOCATION) SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \ CLUSTER=${{ parameters.clusterName }}-$(commitID) \ @@ -49,6 +43,7 @@ stages: cnsVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cnsVersion'] ] cniVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cniVersion'] ] kernelType: $[ stagedependencies.setup.env.outputs['SetEnvVars.kernelType'] ] + ipamVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.ipamVersion'] ] pool: name: $(BUILD_POOL_NAME_DEFAULT) dependsOn: @@ -56,27 +51,66 @@ stages: - setup displayName: "CNIv2 Test - ${{ parameters.name }}" jobs: - - job: integration - displayName: "Integration Test - ${{ parameters.name }}" - steps: - - task: AzureCLI@1 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - echo cns version - $(cnsVersion) - echo cni version - $(cniVersion) + - ${{ if contains(parameters.name, 'linux') }}: + - job: integration + displayName: "Integration Test - ${{ parameters.name }}" + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo cns version - $(cnsVersion) + echo cni version - $(cniVersion) - echo "Start Integration Tests on Overlay Cluster" - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - kubectl cluster-info - kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(cnsVersion) CNI_VERSION=$(cniVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) CNI_IMAGE_REPO=$(CNI_IMAGE_REPO) - name: "overlaye2e" - displayName: "Overlay Integration" - retryCountOnTaskFailure: 2 + echo "Start Integration Tests on Overlay Cluster" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + kubectl cluster-info + kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=$(cnsVersion) CNI_VERSION=$(cniVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) CNI_IMAGE_REPO=$(CNI_IMAGE_REPO) + name: "overlaye2e" + displayName: "Overlay Integration" + retryCountOnTaskFailure: 2 + - ${{ if contains(parameters.name, 'cilium') }}: + - job: integration + displayName: "Deploy Cilium Components" + steps: + - task: AzureCLI@1 + displayName: "Install Cilium, CNS, and ip-masq-agent" + retryCountOnTaskFailure: 2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + az extension add --name aks-preview + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + + echo "install Cilium ${CILIUM_VERSION_TAG}" + export DIR=${CILIUM_VERSION_TAG%.*} + echo "installing files from ${DIR}" + + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-config/cilium-config.yaml + + # Passes Cilium image to daemonset and deployment + kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-agent/files + kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-operator/files + + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - + kubectl get po -owide -A + echo "Deploy Azure-CNS" + sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true AZURE_IPAM_VERSION=$(ipamVersion) CNS_VERSION=$(cnsVersion) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) + kubectl get po -owide -A - job: kernel_upgrade displayName: "Kernel Node Upgrade" dependsOn: integration @@ -84,8 +118,6 @@ stages: - template: kernel-upgrade-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) - KernelVersion : ${{ parameters.KernelVersion }} - ProposedRepoVersion : ${{ parameters.ProposedRepoVersion }} - job: deploy_pods displayName: "Scale Test" dependsOn: kernel_upgrade @@ -97,12 +129,12 @@ stages: os: ${{ parameters.os }} iterations: ${ITERATIONS_LINUX} nodeCount: ${{ parameters.nodeCount }} - cni: cniv2 + cni: ${{ parameters.cni }} - template: ../load-test-templates/validate-state-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} - cni: cniv2 + cni: ${{ parameters.cni }} - job: restart_nodes displayName: "Restart Test" dependsOn: deploy_pods @@ -111,12 +143,12 @@ stages: parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} - cni: cniv2 + cni: ${{ parameters.cni }} - template: ../load-test-templates/validate-state-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} - cni: cniv2 + cni: ${{ parameters.cni }} restartCase: "true" - job: restart_cns displayName: "Restart and Validate CNS" @@ -126,7 +158,7 @@ stages: parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} - cni: cniv2 + cni: ${{ parameters.cni }} scaleup: ${SCALEUP_LINUX} nodeCount: ${{ parameters.nodeCount }} - job: recover @@ -149,17 +181,71 @@ stages: kubectl get po -owide -A name: "recover" displayName: "Delete test Namespaces" - - template: ../k8s-e2e/k8s-e2e-job-template.yaml - parameters: - sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: ${{ parameters.os }} - dependsOn: recover - datapath: true - dns: true - portforward: true - service: true - hostport: true + - job: cni_tests + displayName: "Cilium Test" + dependsOn: recover + condition: and( succeeded(), ${{ contains(parameters.cni, 'cilium') }} ) + steps: + - script: | + echo "install cilium CLI" + if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then + echo "Cilium Agent Version ${BASH_REMATCH[0]}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt) + else + echo "Cilium Agent Version ${CILIUM_VERSION_TAG}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) + fi + CLI_ARCH=amd64 + curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum + sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin + rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + cilium status + cilium version + name: "InstallCiliumCli" + displayName: "Install Cilium CLI" + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + name: "GetCluster" + displayName: "Get AKS Cluster" + - script: | + kubectl delete ns load-test + cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' + retryCountOnTaskFailure: 6 + name: "CiliumConnectivityTests" + displayName: "Run Cilium Connectivity Tests" + - ${{ if contains(parameters.cni, 'cilium') }}: + - template: ../k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + dependsOn: cni_tests + datapath: true + dns: true + portforward: true + service: true + - ${{ if contains(parameters.cni, 'cniv2') }}: + - template: ../k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + dependsOn: recover + datapath: true + dns: true + portforward: true + service: true + hostport: true - job: failedE2ELogs displayName: "Failure Logs" dependsOn: @@ -169,6 +255,7 @@ stages: - restart_nodes - restart_cns - recover + - cni_tests - cni_${{ parameters.os }} condition: failed() steps: @@ -176,4 +263,4 @@ stages: parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} - cni: cniv2 + cni: ${{ parameters.cni }} diff --git a/.pipelines/cni/lsg/pipeline.yaml b/.pipelines/cni/lsg/pipeline.yaml index cbddeba6c2..2001926583 100644 --- a/.pipelines/cni/lsg/pipeline.yaml +++ b/.pipelines/cni/lsg/pipeline.yaml @@ -1,14 +1,3 @@ -parameters: -- name: KernelVersion - type: string - default: 1000 -- name: KernelType - type: string - default: linux-azure-edge -- name: ProposedRepoVersion - type: string - default: proposed - pr: none trigger: none @@ -36,41 +25,50 @@ stages: echo "##vso[task.setvariable variable=commitID;isOutput=true]$(echo $(make revision)-$(date "+%d%H%M"))" echo "##vso[task.setvariable variable=cnsVersion;isOutput=true]$(CNS_VERSION)" echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(CNI_VERSION)" + echo "##vso[task.setvariable variable=ipamVersion;isOutput=true]$(IPAM_VERSION)" echo "------" echo Queued by $(Build.QueuedBy) - echo "Runtime Parameters passed" - echo "KernelType : ${{ parameters.KernelType }}" - echo "KernelVersion : ${{ parameters.KernelVersion }}" - echo "ProposedRepoVersion : ${{ parameters.ProposedRepoVersion }}" + echo "Runtime Variables passed" + echo "KernelType : $(KernelType)" + echo "KernelVersion : $(KernelVersion)" + echo "ProposedRepoVersion : $(ProposedRepoVersion)" echo "------" echo "KernelType Check" - KernelType=${{ parameters.KernelType }} - if [ $KernelType = 'proposed-azure' ]; then - echo "KernelType is $KernelType , change to linux-azure" - KernelType=linux-azure + Kernel=$(KernelType) + if [ $Kernel = 'proposed-azure' ]; then + echo "KernelType is $Kernel , change to linux-azure" + Kernel=linux-azure fi - if [ $KernelType = 'proposed-edge' ]; then - echo "KernelType is $KernelType , change to linux-azure-edge" - KernelType=linux-azure-edge + if [ $Kernel = 'proposed-edge' ]; then + echo "KernelType is $Kernel , change to linux-azure-edge" + Kernel=linux-azure-edge fi - echo "##vso[task.setvariable variable=kernelType;isOutput=true]$KernelType" + echo "Final KernelType : $Kernel" + echo "##vso[task.setvariable variable=kernelType;isOutput=true]$Kernel" name: "SetEnvVars" displayName: "Set Environment Variables" condition: always() - template: lsg-cni-intergration-template.yaml parameters: - name: linux_over + name: linux_overlay clusterType: overlay-byocni-up clusterName: "kup-over" nodeCount: 2 vmSize: Standard_B2ms arch: amd64 cni: cniv2 - test: par - KernelVersion : ${{ parameters.KernelVersion }} - ProposedRepoVersion : ${{ parameters.ProposedRepoVersion }} + + - template: lsg-cni-intergration-template.yaml + parameters: + name: cilium_overlay + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "kup-cilover" + nodeCount: 2 + vmSize: Standard_B2ms + arch: amd64 + cni: cilium - stage: delete_resources displayName: "Delete Resources" @@ -78,7 +76,8 @@ stages: name: "$(BUILD_POOL_NAME_DEFAULT)" condition: always() dependsOn: - - linux_over + - linux_overlay + - cilium_overlay - setup variables: commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ] @@ -90,8 +89,11 @@ stages: strategy: matrix: linux_overlay: - name: linux_over + name: linux_overlay clusterName: "kup-over" + cilium_overlay: + name: cilium_overlay + clusterName: "kup-cilover" steps: - template: ../../templates/delete-cluster.yaml parameters: diff --git a/test/internal/kubernetes/utils_create.go b/test/internal/kubernetes/utils_create.go index 03707e0cac..54ef5003c4 100644 --- a/test/internal/kubernetes/utils_create.go +++ b/test/internal/kubernetes/utils_create.go @@ -54,6 +54,7 @@ const ( envCNSVersion = "CNS_VERSION" envCNIImageRepo = "CNI_IMAGE_REPO" envCNSImageRepo = "CNS_IMAGE_REPO" + envAzureIPAMImageRepo = "IPAM_IMAGE_REPO" EnvInstallCNS = "INSTALL_CNS" cnsLinuxLabelSelector = "k8s-app=azure-cns" cnsWindowsLabelSelector = "k8s-app=azure-cns-win" @@ -345,7 +346,14 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error) } initContainerNameCNI := path.Join(url, "azure-cni:") + os.Getenv(envCNIVersion) log.Printf("CNI init container image - %v", initContainerNameCNI) - initContainerNameIPAM := "acnpublic.azurecr.io/azure-ipam:" + os.Getenv(envAzureIPAMVersion) + + url, key = imageRepoURL[os.Getenv(string(envAzureIPAMImageRepo))] + if !key { + log.Printf("%s not set to expected value \"ACN\", \"MCR\". Default to %s", envAzureIPAMImageRepo, imageRepoURL["ACN"]) + url = imageRepoURL["ACN"] + } + initContainerNameIPAM := path.Join(url, "azure-ipam:") + os.Getenv(envAzureIPAMVersion) + log.Printf("IPAM init container image - %v", initContainerNameIPAM) // cns scenario map cnsScenarioMap := map[CNSScenario]map[corev1.OSName]cnsDetails{