feat: configurable agent/work pool limit (#189)

This allows users to configure the maximum number of simultaneous flow runs that an agent or worker will start. For details, see https://docs.prefect.io/latest/concepts/work-pools/#agent-options or https://docs.prefect.io/latest/concepts/work-pools/#worker-options Closes: #181
PrefectHQ · Jun 29, 2023 · 5a55a14 · 5a55a14
1 parent 0a74a14
commit 5a55a14
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 3 deletions.
diff --git a/charts/prefect-agent/README.md b/charts/prefect-agent/README.md
@@ -38,6 +38,7 @@ Prefect Agent application bundle
 | agent.cloudApiConfig.workspaceId | string | `""` | prefect workspace ID |
 | agent.clusterUid | string | `""` | unique cluster identifier, if none is provided this value will be infered at time of helm install |
 | agent.config.http2 | bool | `true` | connect using HTTP/2 if the server supports it (experimental) |
+| agent.config.limit | string | `"None"` | Maximum number of flow runs to start simultaneously (default: unlimited) |
 | agent.config.prefetchSeconds | int | `10` | when querying for runs, how many seconds in the future can they be scheduled |
 | agent.config.queryInterval | int | `5` | how often the agent will query for runs |
 | agent.config.workPool | string | `""` | name of prefect workpool the agent will poll; if workpool or workqueues is not provided, we use the default queue |

diff --git a/charts/prefect-agent/templates/deployment.yaml b/charts/prefect-agent/templates/deployment.yaml
@@ -77,6 +77,8 @@ spec:
             - --work-queue
             - "default"
             {{- end }}
+            - --limit
+            - {{ .Values.agent.config.limit | quote }}
           workingDir: /home/prefect
           env:
             - name: HOME

diff --git a/charts/prefect-agent/values.yaml b/charts/prefect-agent/values.yaml
@@ -49,6 +49,8 @@ agent:
     prefetchSeconds: 10
     # -- connect using HTTP/2 if the server supports it (experimental)
     http2: true
+    # -- Maximum number of flow runs to start simultaneously (default: unlimited)
+    limit: None
 
   ## connection settings
   # -- one of 'cloud' or 'server'

diff --git a/charts/prefect-worker/README.md b/charts/prefect-worker/README.md
@@ -1,7 +1,6 @@
 # prefect-worker
 
-
-![Version: 0.0.0](https://img.shields.io/badge/Version-0.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2-latest](https://img.shields.io/badge/AppVersion-2--latest-informational?style=flat-square) 
+![Version: 0.0.0](https://img.shields.io/badge/Version-0.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2-latest](https://img.shields.io/badge/AppVersion-2--latest-informational?style=flat-square)
 
 Prefect Worker application bundle
 
@@ -52,6 +51,7 @@ Prefect Worker application bundle
 | worker.cloudApiConfig.workspaceId | string | `""` | prefect workspace ID |
 | worker.clusterUid | string | `""` | unique cluster identifier, if none is provided this value will be infered at time of helm install |
 | worker.config.http2 | bool | `true` | connect using HTTP/2 if the server supports it (experimental) |
+| worker.config.limit | string | `"None"` | Maximum number of flow runs to start simultaneously (default: unlimited) |
 | worker.config.prefetchSeconds | int | `10` | when querying for runs, how many seconds in the future can they be scheduled |
 | worker.config.queryInterval | int | `5` | how often the worker will query for runs |
 | worker.config.workPool | string | `""` | name of prefect work pool the worker will poll |
@@ -89,6 +89,5 @@ Prefect Worker application bundle
 | worker.serverApiConfig.apiUrl | string | `"http://127.0.0.1:4200/api"` | prefect API url (PREFECT_API_URL); should be in-cluster URL if the worker is deployed in the same cluster as the API |
 | worker.tolerations | list | `[]` | tolerations for worker pods assignment |
 
-
 ----------------------------------------------
 Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
diff --git a/charts/prefect-worker/templates/deployment.yaml b/charts/prefect-worker/templates/deployment.yaml
@@ -68,6 +68,8 @@ spec:
             - kubernetes
             - --pool
             - {{ .Values.worker.config.workPool | quote }}
+            - --limit
+            - {{ .Values.worker.config.limit | quote }}
             {{- if .Values.worker.livenessProbe.enabled }}
             - --with-healthcheck
             {{- end }}

diff --git a/charts/prefect-worker/values.yaml b/charts/prefect-worker/values.yaml
@@ -45,6 +45,8 @@ worker:
     prefetchSeconds: 10
     # -- connect using HTTP/2 if the server supports it (experimental)
     http2: true
+    # -- Maximum number of flow runs to start simultaneously (default: unlimited)
+    limit: None
 
   ## connection settings
   # -- one of 'cloud' or 'server'