Merge pull request #1458 from thapabishwa/feature/env-from

feat: add envFrom support for fluentd daemonset and statefulsets
fluent · Jan 30, 2025 · bda6ebd · bda6ebd
2 parents 9798273 + 9dd2abf
commit bda6ebd
Show file tree

Hide file tree

Showing 17 changed files with 241 additions and 3 deletions.
diff --git a/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/input/zz_generated.deepcopy.go
diff --git a/apis/fluentbit/v1alpha2/plugins/output/kafka_types.go b/apis/fluentbit/v1alpha2/plugins/output/kafka_types.go
@@ -47,7 +47,7 @@ type Kafka struct {
 	// Limit the maximum number of Chunks in the filesystem for the current output logical destination.
 	TotalLimitSize string `json:"totalLimitSize,omitempty"`
 	// Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0.
-	Workers      *int32 `json:"workers,omitempty"`
+	Workers *int32 `json:"workers,omitempty"`
 }
 
 func (*Kafka) Name() string {

diff --git a/apis/fluentbit/v1alpha2/plugins/output/loki_types.go b/apis/fluentbit/v1alpha2/plugins/output/loki_types.go
@@ -62,7 +62,7 @@ type Loki struct {
 	// Limit the maximum number of Chunks in the filesystem for the current output logical destination.
 	TotalLimitSize string `json:"totalLimitSize,omitempty"`
 	// Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0.
-	Workers      *int32 `json:"workers,omitempty"`
+	Workers *int32 `json:"workers,omitempty"`
 }
 
 // implement Section() method

diff --git a/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go b/apis/fluentbit/v1alpha2/plugins/output/zz_generated.deepcopy.go
diff --git a/apis/fluentd/v1alpha1/fluentd_types.go b/apis/fluentd/v1alpha1/fluentd_types.go
@@ -61,6 +61,8 @@ type FluentdSpec struct {
 	Args []string `json:"args,omitempty"`
 	// EnvVars represent environment variables that can be passed to fluentd pods.
 	EnvVars []corev1.EnvVar `json:"envVars,omitempty"`
+	// EnvFrom represent environment variables that can be passed to fluentd pods directly from secret or configmap
+	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
 	// FluentdCfgSelector defines the selectors to select the fluentd config CRs.
 	FluentdCfgSelector metav1.LabelSelector `json:"fluentdCfgSelector,omitempty"`
 	// Buffer definition

diff --git a/apis/fluentd/v1alpha1/zz_generated.deepcopy.go b/apis/fluentd/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml b/charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_fluentds.yaml
@@ -1781,6 +1781,56 @@ spec:
                 description: By default will build the related service according to
                   the globalinputs definition.
                 type: boolean
+              envFrom:
+                description: EnvFrom represent environment variables that can be passed
+                  to fluentd pods directly from secret or configmap
+                items:
+                  description: EnvFromSource represents the source of a set of ConfigMaps
+                  properties:
+                    configMapRef:
+                      description: The ConfigMap to select from
+                      properties:
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            TODO: Add other useful fields. apiVersion, kind, uid?
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                          type: string
+                        optional:
+                          description: Specify whether the ConfigMap must be defined
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    prefix:
+                      description: An optional identifier to prepend to each key in
+                        the ConfigMap. Must be a C_IDENTIFIER.
+                      type: string
+                    secretRef:
+                      description: The Secret to select from
+                      properties:
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            TODO: Add other useful fields. apiVersion, kind, uid?
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                          type: string
+                        optional:
+                          description: Specify whether the Secret must be defined
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                  type: object
+                type: array
               envVars:
                 description: EnvVars represent environment variables that can be passed
                   to fluentd pods.

diff --git a/config/crd/bases/fluentd.fluent.io_fluentds.yaml b/config/crd/bases/fluentd.fluent.io_fluentds.yaml
@@ -1781,6 +1781,56 @@ spec:
                 description: By default will build the related service according to
                   the globalinputs definition.
                 type: boolean
+              envFrom:
+                description: EnvFrom represent environment variables that can be passed
+                  to fluentd pods directly from secret or configmap
+                items:
+                  description: EnvFromSource represents the source of a set of ConfigMaps
+                  properties:
+                    configMapRef:
+                      description: The ConfigMap to select from
+                      properties:
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            TODO: Add other useful fields. apiVersion, kind, uid?
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                          type: string
+                        optional:
+                          description: Specify whether the ConfigMap must be defined
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    prefix:
+                      description: An optional identifier to prepend to each key in
+                        the ConfigMap. Must be a C_IDENTIFIER.
+                      type: string
+                    secretRef:
+                      description: The Secret to select from
+                      properties:
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            TODO: Add other useful fields. apiVersion, kind, uid?
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                          type: string
+                        optional:
+                          description: Specify whether the Secret must be defined
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                  type: object
+                type: array
               envVars:
                 description: EnvVars represent environment variables that can be passed
                   to fluentd pods.

diff --git a/docs/fluentd.md b/docs/fluentd.md
@@ -338,6 +338,7 @@ FluentdSpec defines the desired state of Fluentd
 | image | Fluentd image. | string |
 | args | Fluentd Watcher command line arguments. | []string |
 | envVars | EnvVars represent environment variables that can be passed to fluentd pods. | [][corev1.EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#envvar-v1-core) |
+| envFrom | EnvFrom represent environment variables that can be passed to fluentd pods directly from secret or configmap | [][corev1.EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#envfromsource-v1-core) |
 | fluentdCfgSelector | FluentdCfgSelector defines the selectors to select the fluentd config CRs. | [metav1.LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#labelselector-v1-meta) |
 | buffer | Buffer definition | *[BufferVolume](#buffervolume) |
 | imagePullPolicy | Fluentd image pull policy. | [corev1.PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#pullpolicy-v1-core) |

diff --git a/docs/plugins/fluentbit/input/tail.md b/docs/plugins/fluentbit/input/tail.md
@@ -35,3 +35,4 @@ The Tail input plugin allows to monitor one or several text files. <br /> It has
 | storageType | Specify the buffering mechanism to use. It can be memory or filesystem | string |
 | pauseOnChunksOverlimit | Specifies if the input plugin should be paused (stop ingesting new data) when the storage.max_chunks_up value is reached. | string |
 | skipEmptyLines | Skips empty lines in the log file from any further processing or output. | *bool |
+| threaded | Threaded mechanism allows input plugin to run in a separate thread which helps to desaturate the main pipeline. | *string |
diff --git a/docs/plugins/fluentbit/output/kafka.md b/docs/plugins/fluentbit/output/kafka.md
@@ -16,3 +16,5 @@ Kafka output plugin allows to ingest your records into an Apache Kafka service.
 | rdkafka | {property} can be any librdkafka properties | map[string]string |
 | dynamicTopic | adds unknown topics (found in Topic_Key) to Topics. So in Topics only a default topic needs to be configured | *bool |
 | queueFullRetries | Fluent Bit queues data into rdkafka library, if for some reason the underlying library cannot flush the records the queue might fills up blocking new addition of records. The queue_full_retries option set the number of local retries to enqueue the data. The default value is 10 times, the interval between each retry is 1 second. Setting the queue_full_retries value to 0 set's an unlimited number of retries. | *int64 |
+| totalLimitSize | Limit the maximum number of Chunks in the filesystem for the current output logical destination. | string |
+| workers | Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. | *int32 |
diff --git a/docs/plugins/fluentbit/output/loki.md b/docs/plugins/fluentbit/output/loki.md
@@ -22,3 +22,5 @@ The loki output plugin, allows to ingest your records into a Loki service. <br /
 | tenantIDKey | Specify the name of the key from the original record that contains the Tenant ID. The value of the key is set as X-Scope-OrgID of HTTP header. It is useful to set Tenant ID dynamically. | string |
 | tls |  | *[plugins.TLS](../tls.md) |
 | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking |
+| totalLimitSize | Limit the maximum number of Chunks in the filesystem for the current output logical destination. | string |
+| workers | Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. | *int32 |
diff --git a/docs/plugins/fluentbit/output/open_search.md b/docs/plugins/fluentbit/output/open_search.md
@@ -36,7 +36,7 @@ OpenSearch is the opensearch output plugin, allows to ingest your records into a
 | currentTimeIndex | Use current time for index generation instead of message record | *bool |
 | logstashPrefixKey | Prefix keys with this string | string |
 | suppressTypeName | When enabled, mapping types is removed and Type option is ignored. Types are deprecated in APIs in v7.0. This options is for v7.0 or later. | *bool |
-| Workers | Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. | *int32 |
+| workers | Enables dedicated thread(s) for this output. Default value is set since version 1.8.13. For previous versions is 0. | *int32 |
 | tls |  | *[plugins.TLS](../tls.md) |
 | networking | Include fluentbit networking options for this output-plugin | *plugins.Networking |
 | totalLimitSize | Limit the maximum number of Chunks in the filesystem for the current output logical destination. | string |

diff --git a/manifests/setup/fluent-operator-crd.yaml b/manifests/setup/fluent-operator-crd.yaml
@@ -25963,6 +25963,56 @@ spec:
                 description: By default will build the related service according to
                   the globalinputs definition.
                 type: boolean
+              envFrom:
+                description: EnvFrom represent environment variables that can be passed
+                  to fluentd pods directly from secret or configmap
+                items:
+                  description: EnvFromSource represents the source of a set of ConfigMaps
+                  properties:
+                    configMapRef:
+                      description: The ConfigMap to select from
+                      properties:
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            TODO: Add other useful fields. apiVersion, kind, uid?
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                          type: string
+                        optional:
+                          description: Specify whether the ConfigMap must be defined
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    prefix:
+                      description: An optional identifier to prepend to each key in
+                        the ConfigMap. Must be a C_IDENTIFIER.
+                      type: string
+                    secretRef:
+                      description: The Secret to select from
+                      properties:
+                        name:
+                          default: ""
+                          description: |-
+                            Name of the referent.
+                            This field is effectively required, but due to backwards compatibility is
+                            allowed to be empty. Instances of this type with an empty value here are
+                            almost certainly wrong.
+                            TODO: Add other useful fields. apiVersion, kind, uid?
+                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
+                          type: string
+                        optional:
+                          description: Specify whether the Secret must be defined
+                          type: boolean
+                      type: object
+                      x-kubernetes-map-type: atomic
+                  type: object
+                type: array
               envVars:
                 description: EnvVars represent environment variables that can be passed
                   to fluentd pods.