Skip to content

Commit

Permalink
ci: Add Cilium to CNI | LSG Integration Pipeline (#2574)
Browse files Browse the repository at this point in the history 
* ci: Kernel upgrade template

* ci: lsg cni intergration template

* chore: address comments

* ci: add IPAM repo/ver control

* ci: add cilium

* chore: template style

* fix: DAG logic

* ci: finalize changes

* ci: update cilium filepaths

* ci: address comments
  • Loading branch information
@jpayne3506
jpayne3506 authored Mar 29, 2024
1 parent b7d38ae commit 7c4e48c
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 82 deletions.
12 changes: 6 additions & 6 deletions .pipelines/cni/lsg/kernel-upgrade-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ steps:
privList=`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'`
for pod in $privList; do
echo "-- Add proposed repository --"
kubectl exec -i -n kube-system $pod -- bash -c "add-apt-repository ppa:canonical-kernel-team/${{ parameters.ProposedRepoVersion }} -y"
kubectl exec -i -n kube-system $pod -- bash -c "add-apt-repository ppa:canonical-kernel-team/$(ProposedRepoVersion) -y"
echo "-- Check apt-cache --"
kubectl exec -i -n kube-system $pod -- bash -c "apt-cache madison $(kernelType)"
Expand All @@ -58,12 +59,11 @@ steps:
for pod in $privList; do
flag1=""
flag2=""
echo "-- Confirm Proposed Repository --"
temp=`kubectl exec -i -n kube-system $pod -- bash -c "apt-cache madison $(kernelType)" | grep proposed | awk '{print $3}'`
for t in $temp; do
if [[ $t =~ ${{ parameters.KernelVersion }} ]]; then
echo "$t - looking for ${{ parameters.KernelVersion }}"
if [[ $t =~ $(KernelVersion) ]]; then
echo "$t - looking for $(KernelVersion)"
flag1="true"
fi
done
Expand Down Expand Up @@ -126,8 +126,8 @@ steps:
echo "-- Confirm Ubuntu Kernel --"
kernelOut=`kubectl exec -i -n kube-system $pod -- bash -c "uname -r"`
ver=(${kernelOut//-/ }) # Slices $kernelOut with '-'
echo "$pod | kernel version - $kernelOut, expected - ${{ parameters.KernelVersion }}"
if [[ ! ${{ parameters.KernelVersion }} =~ ${ver[1]} ]]; then
echo "$pod | kernel version - $kernelOut, expected - $(KernelVersion)"
if [[ ! $(KernelVersion) =~ ${ver[1]} ]]; then
exit 1
fi
done
Expand Down
179 changes: 133 additions & 46 deletions .pipelines/cni/lsg/lsg-cni-intergration-template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,6 @@ parameters:
arch: ""
osSKU: Ubuntu

# CNIv2
# + Should be able to scale up/down the pods successfully certain number of times.
# + Node reboot scenarios should be covered.
# + The CNS state should be validated with that of CNI state.
# + Pods should have ips assigned and connectivity/datapath test should be present.
# + CNS restart and validates the state

stages:
- stage: create_${{ parameters.name }}
variables:
Expand All @@ -35,6 +28,7 @@ stages:
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
make -C ./hack/aks ${{ parameters.clusterType }} \
AZCLI=az REGION=$(LOCATION) SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \
CLUSTER=${{ parameters.clusterName }}-$(commitID) \
Expand All @@ -49,43 +43,81 @@ stages:
cnsVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cnsVersion'] ]
cniVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cniVersion'] ]
kernelType: $[ stagedependencies.setup.env.outputs['SetEnvVars.kernelType'] ]
ipamVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.ipamVersion'] ]
pool:
name: $(BUILD_POOL_NAME_DEFAULT)
dependsOn:
- create_${{ parameters.name }}
- setup
displayName: "CNIv2 Test - ${{ parameters.name }}"
jobs:
- job: integration
displayName: "Integration Test - ${{ parameters.name }}"
steps:
- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
echo cns version - $(cnsVersion)
echo cni version - $(cniVersion)
- ${{ if contains(parameters.name, 'linux') }}:
- job: integration
displayName: "Integration Test - ${{ parameters.name }}"
steps:
- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
echo cns version - $(cnsVersion)
echo cni version - $(cniVersion)
echo "Start Integration Tests on Overlay Cluster"
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
kubectl cluster-info
kubectl get po -owide -A
sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(cnsVersion) CNI_VERSION=$(cniVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) CNI_IMAGE_REPO=$(CNI_IMAGE_REPO)
name: "overlaye2e"
displayName: "Overlay Integration"
retryCountOnTaskFailure: 2
echo "Start Integration Tests on Overlay Cluster"
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
kubectl cluster-info
kubectl get po -owide -A
sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true CNS_VERSION=$(cnsVersion) CNI_VERSION=$(cniVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) CNI_IMAGE_REPO=$(CNI_IMAGE_REPO)
name: "overlaye2e"
displayName: "Overlay Integration"
retryCountOnTaskFailure: 2
- ${{ if contains(parameters.name, 'cilium') }}:
- job: integration
displayName: "Deploy Cilium Components"
steps:
- task: AzureCLI@1
displayName: "Install Cilium, CNS, and ip-masq-agent"
retryCountOnTaskFailure: 2
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -ex
az extension add --name aks-preview
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
ls -lah
pwd
kubectl cluster-info
kubectl get po -owide -A
echo "install Cilium ${CILIUM_VERSION_TAG}"
export DIR=${CILIUM_VERSION_TAG%.*}
echo "installing files from ${DIR}"
echo "deploy Cilium ConfigMap"
kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-config/cilium-config.yaml
# Passes Cilium image to daemonset and deployment
kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-agent/files
kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-operator/files
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f -
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f -
kubectl get po -owide -A
echo "Deploy Azure-CNS"
sudo -E env "PATH=$PATH" make test-load CNS_ONLY=true AZURE_IPAM_VERSION=$(ipamVersion) CNS_VERSION=$(cnsVersion) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO)
kubectl get po -owide -A
- job: kernel_upgrade
displayName: "Kernel Node Upgrade"
dependsOn: integration
steps:
- template: kernel-upgrade-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}-$(commitID)
KernelVersion : ${{ parameters.KernelVersion }}
ProposedRepoVersion : ${{ parameters.ProposedRepoVersion }}
- job: deploy_pods
displayName: "Scale Test"
dependsOn: kernel_upgrade
Expand All @@ -97,12 +129,12 @@ stages:
os: ${{ parameters.os }}
iterations: ${ITERATIONS_LINUX}
nodeCount: ${{ parameters.nodeCount }}
cni: cniv2
cni: ${{ parameters.cni }}
- template: ../load-test-templates/validate-state-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: cniv2
cni: ${{ parameters.cni }}
- job: restart_nodes
displayName: "Restart Test"
dependsOn: deploy_pods
Expand All @@ -111,12 +143,12 @@ stages:
parameters:
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: cniv2
cni: ${{ parameters.cni }}
- template: ../load-test-templates/validate-state-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: cniv2
cni: ${{ parameters.cni }}
restartCase: "true"
- job: restart_cns
displayName: "Restart and Validate CNS"
Expand All @@ -126,7 +158,7 @@ stages:
parameters:
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: cniv2
cni: ${{ parameters.cni }}
scaleup: ${SCALEUP_LINUX}
nodeCount: ${{ parameters.nodeCount }}
- job: recover
Expand All @@ -149,17 +181,71 @@ stages:
kubectl get po -owide -A
name: "recover"
displayName: "Delete test Namespaces"
- template: ../k8s-e2e/k8s-e2e-job-template.yaml
parameters:
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
dependsOn: recover
datapath: true
dns: true
portforward: true
service: true
hostport: true
- job: cni_tests
displayName: "Cilium Test"
dependsOn: recover
condition: and( succeeded(), ${{ contains(parameters.cni, 'cilium') }} )
steps:
- script: |
echo "install cilium CLI"
if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then
echo "Cilium Agent Version ${BASH_REMATCH[0]}"
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt)
else
echo "Cilium Agent Version ${CILIUM_VERSION_TAG}"
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
fi
CLI_ARCH=amd64
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
cilium status
cilium version
name: "InstallCiliumCli"
displayName: "Install Cilium CLI"
- task: AzureCLI@1
inputs:
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
scriptLocation: "inlineScript"
scriptType: "bash"
addSpnToEnvironment: true
inlineScript: |
set -ex
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
name: "GetCluster"
displayName: "Get AKS Cluster"
- script: |
kubectl delete ns load-test
cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption'
retryCountOnTaskFailure: 6
name: "CiliumConnectivityTests"
displayName: "Run Cilium Connectivity Tests"
- ${{ if contains(parameters.cni, 'cilium') }}:
- template: ../k8s-e2e/k8s-e2e-job-template.yaml
parameters:
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: ${{ parameters.cni }}
dependsOn: cni_tests
datapath: true
dns: true
portforward: true
service: true
- ${{ if contains(parameters.cni, 'cniv2') }}:
- template: ../k8s-e2e/k8s-e2e-job-template.yaml
parameters:
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: ${{ parameters.cni }}
dependsOn: recover
datapath: true
dns: true
portforward: true
service: true
hostport: true
- job: failedE2ELogs
displayName: "Failure Logs"
dependsOn:
Expand All @@ -169,11 +255,12 @@ stages:
- restart_nodes
- restart_cns
- recover
- cni_tests
- cni_${{ parameters.os }}
condition: failed()
steps:
- template: ../../templates/log-template.yaml
parameters:
clusterName: ${{ parameters.clusterName }}-$(commitID)
os: ${{ parameters.os }}
cni: cniv2
cni: ${{ parameters.cni }}
Loading

0 comments on commit 7c4e48c

Please sign in to comment.