Skip to content

Commit

Permalink
Merge pull request #1462 from penekk/add-fluentd-elasticsearch-parame…
Browse files Browse the repository at this point in the history 
…ters

feat: Add support for reload_after, sniffer_class_name es output parameters
  • Loading branch information
@wenchajun
wenchajun authored Feb 7, 2025
2 parents dd91ab5 + 68171fe commit 6c6016a
Show file tree
Hide file tree
Showing 9 changed files with 174 additions and 0 deletions.
4 changes: 4 additions & 0 deletions apis/fluentd/v1alpha1/plugins/output/es.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,17 @@ type ElasticsearchCommon struct {
FailOnPuttingTemplateRetryExceeded *bool `json:"failOnPuttingTemplateRetryExceeded,omitempty"`
// Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false)
ReconnectOnError *bool `json:"reconnectOnError,omitempty"`
// Optional, When ReloadConnections true, this is the integer number of operations after which the plugin will reload the connections. The default value is 10000.
ReloadAfter *uint32 `json:"reloadAfter,omitempty"`
// Optional, Automatically reload connection after 10000 documents (default: true)
ReloadConnections *bool `json:"reloadConnections,omitempty"`
// Optional, Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the request, this can be useful to quickly remove a dead node from the list of addresses (default: false)
ReloadOnFailure *bool `json:"reloadOnFailure,omitempty"`
// Optional, HTTP Timeout (default: 5)
// +kubebuilder:validation:Pattern:="^\\d+(s|m|h|d)$"
RequestTimeout *string `json:"requestTimeout,omitempty"`
// Optional, Provide a different sniffer class name
SnifferClassName *string `json:"snifferClassName,omitempty"`
// Optional, Suppress '[types removal]' warnings on elasticsearch 7.x
SuppressTypeName *bool `json:"suppressTypeName,omitempty"`
// Optional, Enable Index Lifecycle Management (ILM)
Expand Down
8 changes: 8 additions & 0 deletions apis/fluentd/v1alpha1/plugins/output/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,10 @@ func (o *Output) elasticsearchPluginCommon(common *ElasticsearchCommon, parent *
parent.InsertPairs("reconnect_on_error", fmt.Sprint(*common.ReconnectOnError))
}

if common.ReloadAfter != nil {
parent.InsertPairs("reload_after", fmt.Sprint(*common.ReloadAfter))
}

if common.ReloadConnections != nil {
parent.InsertPairs("reload_connections", fmt.Sprint(*common.ReloadConnections))
}
Expand All @@ -479,6 +483,10 @@ func (o *Output) elasticsearchPluginCommon(common *ElasticsearchCommon, parent *
parent.InsertPairs("request_timeout", fmt.Sprint(*common.RequestTimeout))
}

if common.SnifferClassName != nil {
parent.InsertPairs("sniffer_class_name", fmt.Sprint(*common.SnifferClassName))
}

if common.SuppressTypeName != nil {
parent.InsertPairs("suppress_type_name", fmt.Sprint(*common.SuppressTypeName))
}
Expand Down
20 changes: 20 additions & 0 deletions charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -822,6 +828,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -1118,6 +1128,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -1137,6 +1153,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down
20 changes: 20 additions & 0 deletions charts/fluent-operator/charts/fluentd-crds/crds/fluentd.fluent.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -822,6 +828,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -1118,6 +1128,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -1137,6 +1153,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down
20 changes: 20 additions & 0 deletions config/crd/bases/fluentd.fluent.io_clusteroutputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -822,6 +828,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -1118,6 +1128,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -1137,6 +1153,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down
20 changes: 20 additions & 0 deletions config/crd/bases/fluentd.fluent.io_outputs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -822,6 +828,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -1118,6 +1128,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -1137,6 +1153,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down
2 changes: 2 additions & 0 deletions docs/plugins/fluentd/output/es.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ Elasticsearch defines the parameters for out_es output plugin
| maxRetryPuttingTemplate | Optional, You can specify times of retry putting template (default: 10) | *uint32 |
| failOnPuttingTemplateRetryExceeded | Optional, Indicates whether to fail when max_retry_putting_template is exceeded. If you have multiple output plugin, you could use this property to do not fail on fluentd statup (default: false) | *bool |
| reconnectOnError | Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false) | *bool |
| reloadAfter | Optional, When ReloadConnections true, this is the integer number of operations after which the plugin will reload the connections. The default value is 10000. | *uint32 |
| reloadConnections | Optional, Automatically reload connection after 10000 documents (default: true) | *bool |
| reloadOnFailure | Optional, Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the request, this can be useful to quickly remove a dead node from the list of addresses (default: false) | *bool |
| requestTimeout | Optional, HTTP Timeout (default: 5) | *string |
| snifferClassName | Optional, Provide a different sniffer class name | *string |
| suppressTypeName | Optional, Suppress '[types removal]' warnings on elasticsearch 7.x | *bool |
| enableIlm | Optional, Enable Index Lifecycle Management (ILM) | *bool |
| ilmPolicyId | Optional, Specify ILM policy id | *string |
Expand Down
40 changes: 40 additions & 0 deletions manifests/setup/fluent-operator-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9287,6 +9287,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -9306,6 +9312,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -9602,6 +9612,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -9621,6 +9637,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -38097,6 +38117,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -38116,6 +38142,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down Expand Up @@ -38412,6 +38442,12 @@ spec:
reset connection on any error (reconnect on next send)
(default: false)'
type: boolean
reloadAfter:
description: Optional, When ReloadConnections true, this
is the integer number of operations after which the plugin
will reload the connections. The default value is 10000.
format: int32
type: integer
reloadConnections:
description: 'Optional, Automatically reload connection
after 10000 documents (default: true)'
Expand All @@ -38431,6 +38467,10 @@ spec:
description: 'Specify https if your Elasticsearch endpoint
supports SSL (default: http).'
type: string
snifferClassName:
description: Optional, Provide a different sniffer class
name
type: string
sslVerify:
description: Optional, Force certificate validation
type: boolean
Expand Down
Loading

0 comments on commit 6c6016a

Please sign in to comment.