From 17418b8f37a4e7441026506f38e03da18d24fce7 Mon Sep 17 00:00:00 2001 From: Payback159 Date: Wed, 15 Jan 2025 00:11:12 +0100 Subject: [PATCH] add structured authentication conf --- .../group_vars/k8s_cluster/k8s-cluster.yml | 4 +- .../defaults/main/kube-apiserver-auth.yml | 37 -------------- .../control-plane/defaults/main/main.yml | 44 +++++++++++++++++ .../control-plane/tasks/kubeadm-setup.yml | 41 +++++----------- .../apiserver-authentication-config.yaml.j2 | 48 ------------------- .../templates/kubeadm-config.v1beta3.yaml.j2 | 10 ++-- .../templates/kubeadm-config.v1beta4.yaml.j2 | 10 ++-- roles/kubernetes/control-plane/vars/main.yaml | 5 -- 8 files changed, 67 insertions(+), 132 deletions(-) delete mode 100644 roles/kubernetes/control-plane/defaults/main/kube-apiserver-auth.yml delete mode 100644 roles/kubernetes/control-plane/templates/apiserver-authentication-config.yaml.j2 diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 24b65b5d6f3..2be3eae140d 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -42,6 +42,7 @@ credentials_dir: "{{ inventory_dir }}/credentials" # kube_oidc_auth: false # kube_token_auth: false + ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) @@ -117,8 +118,7 @@ kube_network_node_prefix_ipv6: 120 # The port the API Server will be listening on. kube_apiserver_ip: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}" -# https port -kube_apiserver_port: 6443 +kube_apiserver_port: 6443 # (https) # Kube-proxy proxyMode configuration. # Can be ipvs, iptables diff --git a/roles/kubernetes/control-plane/defaults/main/kube-apiserver-auth.yml b/roles/kubernetes/control-plane/defaults/main/kube-apiserver-auth.yml deleted file mode 100644 index cb215aa3aa2..00000000000 --- a/roles/kubernetes/control-plane/defaults/main/kube-apiserver-auth.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -kube_apiserver_structured_auth_jwt_issuers: [] -# Example JWT issuer configuration: -# kube_apiserver_structured_auth_jwt_issuers: -# - issuer: -# url: "https://issuer1.example.com" -# audiences: -# - audience1 -# - audience2 -# audienceMatchPolicy: MatchAny -# claimMappings: -# username: -# expression: "claims.username" -# groups: -# expression: "claims.groups" -# uid: -# expression: "claims.uid" -# userValidationRules: -# - expression: "!user.username.startsWith('system:')" -# message: "username cannot use reserved system: prefix" -# - issuer: -# url: "https://issuer2.example.com" -# discoveryURL: "https://discovery.example.com/.well-known/openid-configuration" -# audiences: -# - audience3 -# - audience4 -# audienceMatchPolicy: MatchAny -# claimMappings: -# username: -# expression: "claims.username" -# groups: -# expression: "claims.groups" -# uid: -# expression: "claims.uid" -# userValidationRules: -# - expression: "!user.username.startsWith('kubespray:')" -# message: "username cannot use reserved kubespray: prefix" diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 6269cd8d7dd..de8c0e6304c 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -140,6 +140,50 @@ kube_api_runtime_config: [] kube_token_auth: false kube_oidc_auth: false +## Structured authentication config +## Structured AuthenticationConfiguration is a new feature in Kubernetes v1.29+ (GA in v1.32) that configures the API server's authentication modes with a structured configuration file. +## Note: The `--authentication-config` and `--oidc-*` flags are mutually exclusive. The two features cannot be used at the same time. +## Docs: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#using-authentication-configuration +## Examples: https://kubernetes.io/blog/2024/04/25/structured-authentication-moves-to-beta/#how-to-use-structured-authentication-configuration +## KEP: https://github.com/kubernetes/enhancements/tree/master/keps/sig-auth/3331-structured-authentication-configuration +kube_apiserver_use_authentication_config_file: false +kube_apiserver_authentication_jwt_provider: [] +#Example +# - issuer: +# url: "https://issuer1.example.com" +# audiences: +# - audience1 +# - audience2 +# audienceMatchPolicy: MatchAny +# claimMappings: +# username: +# expression: "claims.username" +# groups: +# expression: "claims.groups" +# uid: +# expression: "claims.uid" +# userValidationRules: +# - expression: "!user.username.startsWith('system:')" +# message: "username cannot use reserved system: prefix" +# - issuer: +# url: "https://issuer2.example.com" +# discoveryURL: "https://discovery.example.com/.well-known/openid-configuration" +# audiences: +# - audience3 +# - audience4 +# audienceMatchPolicy: MatchAny +# claimMappings: +# username: +# expression: "claims.username" +# groups: +# expression: "claims.groups" +# uid: +# expression: "claims.uid" +# userValidationRules: +# - expression: "!user.username.startsWith('kubespray:')" +# message: "username cannot use reserved kubespray: prefix" + + ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication kube_webhook_token_auth: false kube_webhook_token_auth_url_skip_tls_verify: false diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 2d97085bb0b..bccad518e33 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -82,38 +82,19 @@ mode: "0640" when: kube_apiserver_tracing + - name: Kubeadm | Configure Structured Authentication + copy: + content: "{{ authn_config | to_nice_yaml(indent=2, sort_keys=false) }}" + dest: "{{ kube_config_dir }}/apiserver-authentication-config.yaml" + mode: "0640" vars: - all_kube_apiserver_feature_gates: "{{ kube_apiserver_feature_gates + kube_feature_gates }}" - when: - - not kube_oidc_auth - - ("StructuredAuthenticationConfiguration=true" in all_kube_apiserver_feature_gates and kube_version | regex_replace("^v", "") is version("1.30.0", "<", version_type="semver")) - or - ("StructuredAuthenticationConfiguration=false" not in all_kube_apiserver_feature_gates and kube_version | regex_replace("^v", "") is version("1.30.0", ">=", version_type="semver")) - block: - - name: Kubeadm | Create apiserver authentication config directory - file: - path: "{{ kube_config_dir }}/authentication" - state: directory - mode: "0640" - - - name: Merge additional userValidationRules - set_fact: - kube_apiserver_structured_auth_jwt_issuers: >- - {{ - kube_apiserver_structured_auth_jwt_issuers | map('combine', { - 'userValidationRules': item.userValidationRules + additional_user_validation_rules - }) | list - }} - loop: "{{ kube_apiserver_structured_auth_jwt_issuers }}" - loop_control: - loop_var: item - - - name: Kubeadm | Write apiserver authentication config yaml - template: - src: apiserver-authentication-config.yaml.j2 - dest: "{{ kube_config_dir }}/authentication/apiserver-authentication-config.yaml" - mode: "0640" + authn_config: + apiVersion: apiserver.config.k8s.io/{{ 'v1alpha1' if kube_version is version('v1.30.0', '<') else 'v1beta1' if + kube_version is version('v1.32.0', '<') else 'v1' }} + kind: AuthenticationConfiguration + jwt: "{{ kube_apiserver_authentication_config_jwt_provider }}" + when: kube_apiserver_use_authentication_config_file # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. - name: Set kubeadm_config_api_fqdn define diff --git a/roles/kubernetes/control-plane/templates/apiserver-authentication-config.yaml.j2 b/roles/kubernetes/control-plane/templates/apiserver-authentication-config.yaml.j2 deleted file mode 100644 index a09a82e2f5e..00000000000 --- a/roles/kubernetes/control-plane/templates/apiserver-authentication-config.yaml.j2 +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: apiserver.config.k8s.io/v1beta1 -kind: AuthenticationConfiguration -{% if kube_apiserver_structured_auth_jwt_issuers | length > 0 %} -jwt: -{% for issuer in kube_apiserver_structured_auth_jwt_issuers %} -- issuer: - url: "{{ issuer.issuer.url }}" -{% if issuer.issuer.discoveryURL is defined %} - discoveryURL: "{{ issuer.issuer.discoveryURL }}" -{% endif %} - audiences: -{% for audience in issuer.issuer.audiences %} - - {{ audience }} -{% endfor %} - audienceMatchPolicy: {{ issuer.issuer.audienceMatchPolicy }} -{% if issuer.claimValidationRules is defined %} - claimValidationRules: -{% for validationRule in issuer.claimValidationRules %} - expression: "{{ validationRule.expression }}" - message: "{{ validationRule.message }}" -{% endfor %} -{% endif %} - claimMappings: - username: - expression: "{{ issuer.claimMappings.username.expression }}" - groups: - expression: "{{ issuer.claimMappings.groups.expression }}" - uid: - expression: "{{ issuer.claimMappings.uid.expression }}" -{% if issuer.claimMappings.extra is defined %} - extra: -{% for extra in issuer.claimMappings.extra %} - - key: "{{ extra.key }}" - expression: "{{ extra.expression }}" -{% endfor %} -{% endif %} -{% if issuer.userValidationRules | length > 0 %} - userValidationRules: -{% for rule in issuer.userValidationRules %} - - expression: "{{ rule.expression }}" - message: "{{ rule.message }}" -{% endfor %} -{% endif %} -{% endfor %} -{% else %} -jwt: -{% endif %} diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 index 2717c4b3903..eaec09cb470 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2 @@ -177,8 +177,8 @@ apiServer: oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}" {% endif %} {% endif %} -{% if not kube_oidc_auth %} - authentication-config: {{ kube_config_dir }}/authentication/apiserver-authentication-config.yaml +{% if kube_apiserver_use_authentication_config_file %} + authentication-config: {{ kube_config_dir }}/apiserver-authentication-config.yaml {% endif %} {% if kube_webhook_token_auth | default(false) %} authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml @@ -273,10 +273,10 @@ apiServer: readOnly: false pathType: DirectoryOrCreate {% endif %} -{% if not kube_oidc_auth %} +{% if kube_apiserver_use_authentication_config_file %} - name: structauth - hostPath: {{ kube_config_dir }}/authentication - mountPath: {{ kube_config_dir }}/authentication + hostPath: {{ kube_config_dir }}/apiserver-authentication-config.yaml + mountPath: {{ kube_config_dir }}/apiserver-authentication-config.yaml readOnly: true pathType: DirectoryOrCreate {% endif %} diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 index 319bbdc6b79..25a7c36ee4a 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 @@ -213,9 +213,9 @@ apiServer: value: "{{ kube_oidc_groups_prefix }}" {% endif %} {% endif %} -{% if not kube_oidc_auth %} +{% if kube_apiserver_use_authentication_config_file %} - name: authentication-config - value: "{{ kube_config_dir }}/authentication/apiserver-authentication-config.yaml" + value: "{{ kube_config_dir }}/apiserver-authentication-config.yaml" {% endif %} {% if kube_webhook_token_auth | default(false) %} - name: authentication-token-webhook-config-file @@ -331,10 +331,10 @@ apiServer: readOnly: false pathType: DirectoryOrCreate {% endif %} -{% if not kube_oidc_auth %} +{% if kube_apiserver_use_authentication_config_file %} - name: structauth - hostPath: {{ kube_config_dir }}/authentication - mountPath: {{ kube_config_dir }}/authentication + hostPath: {{ kube_config_dir }}/apiserver-authentication-config.yaml + mountPath: {{ kube_config_dir }}/apiserver-authentication-config.yaml readOnly: true pathType: DirectoryOrCreate {% endif %} diff --git a/roles/kubernetes/control-plane/vars/main.yaml b/roles/kubernetes/control-plane/vars/main.yaml index 6fe2cfe5f23..263ee0b4569 100644 --- a/roles/kubernetes/control-plane/vars/main.yaml +++ b/roles/kubernetes/control-plane/vars/main.yaml @@ -7,8 +7,3 @@ kube_apiserver_admission_plugins_needs_configuration: - PodSecurity - PodNodeSelector - ResourceQuota -additional_user_validation_rules: -- expression: "!user.username.startsWith('system:')" - message: "username cannot used reserved system: prefix" -- expression: "user.groups.all(group, !group.startsWith('system:'))" - message: "groups cannot used reserved system: prefix" \ No newline at end of file