Skip to content

Commit

Permalink
Update Tiltfile with AKS VNet peering and deletion logic
Browse files Browse the repository at this point in the history 
- update aks-as-mgmt scripts with VNet creation and all clusters deletion
- internal LB IP can be set using a env variable
  • Loading branch information
@nawazkh
nawazkh committed Jan 2, 2025
1 parent 2239e3b commit 96bfced
Show file tree
Hide file tree
Showing 8 changed files with 150 additions and 36 deletions.
122 changes: 107 additions & 15 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ settings = {
"capi_version": "v1.8.5",
"caaph_version": "v0.2.5",
"cert_manager_version": "v1.16.1",
"kubernetes_version": "v1.28.3",
"aks_kubernetes_version": "v1.28.3",
"kubernetes_version": "v1.28.15",
"aks_kubernetes_version": "v1.28.15",
"flatcar_version": "3374.2.1",
"azure_location": "eastus",
"control_plane_machine_count": "1",
Expand Down Expand Up @@ -212,10 +212,10 @@ def capz():
yaml = str(kustomizesub("./hack/observability")) # build an observable kind deployment by default

# add extra_args if they are defined
if settings.get("extra_args"):
azure_extra_args = settings.get("extra_args").get("azure")
if settings.get("container_args"):
capz_container_args = settings.get("container_args").get("capz-controller-manager")
yaml_dict = decode_yaml_stream(yaml)
append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", azure_extra_args)
append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", capz_container_args)
yaml = str(encode_yaml_stream(yaml_dict))
yaml = fixup_yaml_empty_arrays(yaml)

Expand Down Expand Up @@ -317,9 +317,14 @@ def flavors():
for template in template_list:
deploy_worker_templates(template, substitutions)

delete_all_workload_clusters = kubectl_cmd + " delete clusters --all --wait=false;"

if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
delete_all_workload_clusters += clear_aks_vnet_peerings()

local_resource(
name = "delete-all-workload-clusters",
cmd = kubectl_cmd + " delete clusters --all --wait=false",
cmd = ["sh", "-ec", delete_all_workload_clusters],
auto_init = False,
trigger_mode = TRIGGER_MODE_MANUAL,
labels = ["flavors"],
Expand Down Expand Up @@ -382,16 +387,35 @@ def deploy_worker_templates(template, substitutions):

yaml = shlex.quote(yaml)
flavor_name = os.path.basename(flavor)
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\""
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); "

apiserver_lb_private_ip = os.getenv("AZURE_INTERNAL_LB_PRIVATE_IP", "")
if "windows-apiserver-ilb" in flavor and apiserver_lb_private_ip == "":
flavor_cmd += "export AZURE_INTERNAL_LB_PRIVATE_IP=\"40.0.11.100\"; "
elif "apiserver-ilb" in flavor and apiserver_lb_private_ip == "":
flavor_cmd += "export AZURE_INTERNAL_LB_PRIVATE_IP=\"30.0.11.100\"; "

flavor_cmd += "export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; "
flavor_cmd += "echo \"Cluster ${CLUSTER_NAME} created, don't forget to delete\"; "

# wait for kubeconfig to be available
flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; chmod 600 ./${CLUSTER_NAME}.kubeconfig; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done"
flavor_cmd += "echo \"Waiting for kubeconfig to be available\"; "
flavor_cmd += "until "+ kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; "
flavor_cmd += kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; "
flavor_cmd += "chmod 600 ./${CLUSTER_NAME}.kubeconfig; "
flavor_cmd += "echo \"Kubeconfig for ${CLUSTER_NAME} created and saved in the local\"; "
flavor_cmd += "echo \"Waiting for ${CLUSTER_NAME} API Server to be accessible\"; "
flavor_cmd += "until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done; "
flavor_cmd += "echo \"API Server of ${CLUSTER_NAME} is accessible\"; "

# copy the kubeadm configmap to the calico-system namespace.
# This is a workaround needed for the calico-node-windows daemonset to be able to run in the calico-system namespace.
if "windows" in flavor_name:
flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done"
flavor_cmd += "; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -"
flavor_cmd += "until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done; "
flavor_cmd += kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; "

if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
flavor_cmd += peer_vnets()

flavor_cmd += get_addons(flavor_name)

Expand All @@ -409,14 +433,15 @@ def get_addons(flavor_name):
if "aks" in flavor_name:
return ""

addon_cmd = "; export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}')"
addon_cmd += "; export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ',')"
addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}"
addon_cmd = "export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}'); "
addon_cmd += "export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ','); "
addon_cmd += helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}"
if "flatcar" in flavor_name: # append caCetDir location to the cloud-provider-azure helm install command for flatcar flavor
addon_cmd += " --set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates"
addon_cmd += "; "

if "azure-cni-v1" in flavor_name:
addon_cmd += "; " + kubectl_cmd + " apply -f ./templates/addons/azure-cni-v1.yaml --kubeconfig ./${CLUSTER_NAME}.kubeconfig"
addon_cmd += kubectl_cmd + " apply -f ./templates/addons/azure-cni-v1.yaml --kubeconfig ./${CLUSTER_NAME}.kubeconfig; "
else:
# install calico
if "ipv6" in flavor_name:
Expand All @@ -425,7 +450,7 @@ def get_addons(flavor_name):
calico_values = "./templates/addons/calico-dual-stack/values.yaml"
else:
calico_values = "./templates/addons/calico/values.yaml"
addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://docs.tigera.io/calico/charts --version ${CALICO_VERSION} calico tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace"
addon_cmd += helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://docs.tigera.io/calico/charts --version ${CALICO_VERSION} calico tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace; "

return addon_cmd

Expand Down Expand Up @@ -454,6 +479,73 @@ def waitforsystem():
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system")
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system")

def peer_vnets():
# TODO: check for az cli to be installed in local
# wait for AKS VNet to be in the state created
peering_cmd = '''
echo \"--------Peering VNETs--------\";
az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180;
export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv);
echo \" 1/8 ${AKS_MGMT_VNET_NAME} found \"; '''

# wait for workload VNet to be created
peering_cmd += '''
az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180;
export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv);
echo \" 2/8 ${CLUSTER_NAME}-vnet found \"; '''

# peer mgmt vnet
peering_cmd += '''
az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none;
az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none;
echo \" 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}\"; '''

# peer workload vnet
peering_cmd += '''
az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none;
az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none;
echo \" 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet\"; '''

# create private DNS zone
peering_cmd += '''
az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none;
az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none;
echo \" 5/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}\"; '''

# link private DNS Zone to workload vnet
peering_cmd += '''
az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none;
az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none;
echo \" 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone\"; '''

# link private DNS Zone to mgmt vnet
peering_cmd += '''
az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none;
az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none;
echo \" 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone\"; '''

# create private DNS zone record
peering_cmd += '''
az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --record-set-name \"@\" --ipv4-address ${AZURE_INTERNAL_LB_PRIVATE_IP} --only-show-errors --output none;
echo \" 8/8 \"@\" private DNS zone record created to point ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com to ${AZURE_INTERNAL_LB_PRIVATE_IP}\"; '''

return peering_cmd

def clear_aks_vnet_peerings():
#
delete_peering_cmd = '''
echo \"--------Clearing AKS MGMT VNETs Peerings--------\";
az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180;
echo \" VNet ${AKS_MGMT_VNET_NAME} found \"; '''

# List all peering names and store them in an array
delete_peering_cmd += '''
PEERING_NAMES=$(az network vnet peering list --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --query \"[].name\" --output tsv);
for PEERING_NAME in ${PEERING_NAMES[@]}; do echo \"Deleting peering: ${PEERING_NAME}\"; az network vnet peering delete --name ${PEERING_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME}; done;
echo \"All VNETs Peerings deleted in ${AKS_MGMT_VNET_NAME}\"; '''

return delete_peering_cmd

##############################
# Actual work happens here
##############################
Expand Down
Loading

0 comments on commit 96bfced

Please sign in to comment.