-
-
Notifications
You must be signed in to change notification settings - Fork 638
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use redis worker for run heartbeats and alerts (#1669)
* Move the task run heartbeats to RedisWorker * Move alerts to redis worker, improving redis worker * Fix typecheck errors * Use single threaded tests for redis worker * Enable/disable the redis workers independently * Remove preview release from PR checks
- Loading branch information
Showing
33 changed files
with
796 additions
and
502 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| import { Worker as RedisWorker } from "@internal/redis-worker"; | ||
| import { Logger } from "@trigger.dev/core/logger"; | ||
| import { z } from "zod"; | ||
| import { env } from "~/env.server"; | ||
| import { logger } from "~/services/logger.server"; | ||
| import { singleton } from "~/utils/singleton"; | ||
| import { DeliverAlertService } from "./services/alerts/deliverAlert.server"; | ||
| import { PerformDeploymentAlertsService } from "./services/alerts/performDeploymentAlerts.server"; | ||
| import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; | ||
|
|
||
| function initializeWorker() { | ||
| const redisOptions = { | ||
| keyPrefix: "common:worker:", | ||
| host: env.COMMON_WORKER_REDIS_HOST, | ||
| port: env.COMMON_WORKER_REDIS_PORT, | ||
| username: env.COMMON_WORKER_REDIS_USERNAME, | ||
| password: env.COMMON_WORKER_REDIS_PASSWORD, | ||
| enableAutoPipelining: true, | ||
| ...(env.COMMON_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), | ||
| }; | ||
|
|
||
| logger.debug(`👨🏭 Initializing common worker at host ${env.COMMON_WORKER_REDIS_HOST}`); | ||
|
|
||
| const worker = new RedisWorker({ | ||
| name: "common-worker", | ||
| redisOptions, | ||
| catalog: { | ||
| "v3.performTaskRunAlerts": { | ||
| schema: z.object({ | ||
| runId: z.string(), | ||
| }), | ||
| visibilityTimeoutMs: 60_000, | ||
| retry: { | ||
| maxAttempts: 3, | ||
| }, | ||
| }, | ||
| "v3.performDeploymentAlerts": { | ||
| schema: z.object({ | ||
| deploymentId: z.string(), | ||
| }), | ||
| visibilityTimeoutMs: 60_000, | ||
| retry: { | ||
| maxAttempts: 3, | ||
| }, | ||
| }, | ||
| "v3.deliverAlert": { | ||
| schema: z.object({ | ||
| alertId: z.string(), | ||
| }), | ||
| visibilityTimeoutMs: 60_000, | ||
| retry: { | ||
| maxAttempts: 3, | ||
| }, | ||
| }, | ||
| }, | ||
| concurrency: { | ||
| workers: env.COMMON_WORKER_CONCURRENCY_WORKERS, | ||
| tasksPerWorker: env.COMMON_WORKER_CONCURRENCY_TASKS_PER_WORKER, | ||
| limit: env.COMMON_WORKER_CONCURRENCY_LIMIT, | ||
| }, | ||
| pollIntervalMs: env.COMMON_WORKER_POLL_INTERVAL, | ||
| immediatePollIntervalMs: env.COMMON_WORKER_IMMEDIATE_POLL_INTERVAL, | ||
| logger: new Logger("CommonWorker", "debug"), | ||
| jobs: { | ||
| "v3.deliverAlert": async ({ payload }) => { | ||
| const service = new DeliverAlertService(); | ||
|
|
||
| return await service.call(payload.alertId); | ||
| }, | ||
| "v3.performDeploymentAlerts": async ({ payload }) => { | ||
| const service = new PerformDeploymentAlertsService(); | ||
|
|
||
| return await service.call(payload.deploymentId); | ||
| }, | ||
| "v3.performTaskRunAlerts": async ({ payload }) => { | ||
| const service = new PerformTaskRunAlertsService(); | ||
| return await service.call(payload.runId); | ||
| }, | ||
| }, | ||
| }); | ||
|
|
||
| if (env.COMMON_WORKER_ENABLED === "true") { | ||
| logger.debug( | ||
| `👨🏭 Starting common worker at host ${env.COMMON_WORKER_REDIS_HOST}, pollInterval = ${env.COMMON_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.COMMON_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.COMMON_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.COMMON_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.COMMON_WORKER_CONCURRENCY_LIMIT}` | ||
| ); | ||
|
|
||
| worker.start(); | ||
| } | ||
|
|
||
| return worker; | ||
| } | ||
|
|
||
| export const commonWorker = singleton("commonWorker", initializeWorker); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| import { Worker as RedisWorker } from "@internal/redis-worker"; | ||
| import { Logger } from "@trigger.dev/core/logger"; | ||
| import { z } from "zod"; | ||
| import { env } from "~/env.server"; | ||
| import { logger } from "~/services/logger.server"; | ||
| import { singleton } from "~/utils/singleton"; | ||
| import { TaskRunHeartbeatFailedService } from "./taskRunHeartbeatFailed.server"; | ||
|
|
||
| function initializeWorker() { | ||
| const redisOptions = { | ||
| keyPrefix: "legacy-run-engine:worker:", | ||
| host: env.LEGACY_RUN_ENGINE_WORKER_REDIS_HOST, | ||
| port: env.LEGACY_RUN_ENGINE_WORKER_REDIS_PORT, | ||
| username: env.LEGACY_RUN_ENGINE_WORKER_REDIS_USERNAME, | ||
| password: env.LEGACY_RUN_ENGINE_WORKER_REDIS_PASSWORD, | ||
| enableAutoPipelining: true, | ||
| ...(env.LEGACY_RUN_ENGINE_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), | ||
| }; | ||
|
|
||
| logger.debug( | ||
| `👨🏭 Initializing legacy run engine worker at host ${env.LEGACY_RUN_ENGINE_WORKER_REDIS_HOST}` | ||
| ); | ||
|
|
||
| const worker = new RedisWorker({ | ||
| name: "legacy-run-engine-worker", | ||
| redisOptions, | ||
| catalog: { | ||
| runHeartbeat: { | ||
| schema: z.object({ | ||
| runId: z.string(), | ||
| }), | ||
| visibilityTimeoutMs: 60_000, | ||
| retry: { | ||
| maxAttempts: 3, | ||
| }, | ||
| }, | ||
| }, | ||
| concurrency: { | ||
| workers: env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_WORKERS, | ||
| tasksPerWorker: env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_TASKS_PER_WORKER, | ||
| limit: env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT, | ||
| }, | ||
| pollIntervalMs: env.LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL, | ||
| immediatePollIntervalMs: env.LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL, | ||
| logger: new Logger("LegacyRunEngineWorker", "debug"), | ||
| jobs: { | ||
| runHeartbeat: async ({ payload }) => { | ||
| const service = new TaskRunHeartbeatFailedService(); | ||
|
|
||
| await service.call(payload.runId); | ||
| }, | ||
| }, | ||
| }); | ||
|
|
||
| if (env.LEGACY_RUN_ENGINE_WORKER_ENABLED === "true") { | ||
| logger.debug( | ||
| `👨🏭 Starting legacy run engine worker at host ${env.LEGACY_RUN_ENGINE_WORKER_REDIS_HOST}, pollInterval = ${env.LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL}, immediatePollInterval = ${env.LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL}, workers = ${env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_WORKERS}, tasksPerWorker = ${env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_TASKS_PER_WORKER}, concurrencyLimit = ${env.LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT}` | ||
| ); | ||
|
|
||
| worker.start(); | ||
| } | ||
|
|
||
| return worker; | ||
| } | ||
|
|
||
| export const legacyRunEngineWorker = singleton("legacyRunEngineWorker", initializeWorker); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,28 @@ | ||
| import { RequeueTaskRunService } from "../requeueTaskRun.server"; | ||
| import { legacyRunEngineWorker } from "../legacyRunEngineWorker.server"; | ||
| import { TaskRunHeartbeatFailedService } from "../taskRunHeartbeatFailed.server"; | ||
| import { VisibilityTimeoutStrategy } from "./types"; | ||
|
|
||
| export class V3VisibilityTimeout implements VisibilityTimeoutStrategy { | ||
| export class V3GraphileVisibilityTimeout implements VisibilityTimeoutStrategy { | ||
| async heartbeat(messageId: string, timeoutInMs: number): Promise<void> { | ||
| await RequeueTaskRunService.enqueue(messageId, new Date(Date.now() + timeoutInMs)); | ||
| await TaskRunHeartbeatFailedService.enqueue(messageId, new Date(Date.now() + timeoutInMs)); | ||
| } | ||
|
|
||
| async cancelHeartbeat(messageId: string): Promise<void> { | ||
| await RequeueTaskRunService.dequeue(messageId); | ||
| await TaskRunHeartbeatFailedService.dequeue(messageId); | ||
| } | ||
| } | ||
|
|
||
| export class V3LegacyRunEngineWorkerVisibilityTimeout implements VisibilityTimeoutStrategy { | ||
| async heartbeat(messageId: string, timeoutInMs: number): Promise<void> { | ||
| await legacyRunEngineWorker.enqueue({ | ||
| id: `heartbeat:${messageId}`, | ||
| job: "runHeartbeat", | ||
| payload: { runId: messageId }, | ||
| availableAt: new Date(Date.now() + timeoutInMs), | ||
| }); | ||
| } | ||
|
|
||
| async cancelHeartbeat(messageId: string): Promise<void> { | ||
| await legacyRunEngineWorker.ack(`heartbeat:${messageId}`); | ||
| } | ||
| } |
Oops, something went wrong.