Run Engine 2.0 (alpha)

.changeset/breezy-turtles-talk.md

@@ -0,0 +1,10 @@
+---
+"@trigger.dev/react-hooks": patch
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/build": patch
+"@trigger.dev/core": patch
+"@trigger.dev/rsc": patch
+---
+
+Run Engine 2.0 (alpha)

.configs/tsconfig.base.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "target": "es2022",
-    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
     "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "moduleDetection": "force",

.github/workflows/e2e.yml

@@ -38,6 +38,9 @@ jobs:
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile --filter trigger.dev...
 
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
       - name: 🔧 Build v3 cli monorepo dependencies
         run: pnpm run build --filter trigger.dev^...
 

.github/workflows/pr_checks.yml

@@ -54,6 +54,9 @@ jobs:
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
       - name: 🏗️ Build
         run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev"
 

.github/workflows/unit-tests.yml

@@ -24,6 +24,13 @@
           node-version: 20.11.1
           cache: "pnpm"
 
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: 📥 Download deps
         run: pnpm install --frozen-lockfile
 

.npmrc

@@ -1,3 +1,4 @@
 link-workspace-packages=false
 public-hoist-pattern[]=*prisma*
-prefer-workspace-packages=true
+prefer-workspace-packages=true
+update-notifier=false

.vscode/launch.json

@@ -116,6 +116,14 @@
       "command": "pnpm exec trigger dev",
       "cwd": "${workspaceFolder}/references/hello-world",
       "sourceMaps": true
+    },
+    {
+      "type": "node-terminal",
+      "request": "launch",
+      "name": "Debug RunEngine tests",
+      "command": "pnpm run test --filter @internal/run-engine",
+      "cwd": "${workspaceFolder}",
+      "sourceMaps": true
     }
   ]
 }

apps/coordinator/package.json

@@ -23,10 +23,8 @@
     "tinyexec": "^0.3.0"
   },
   "devDependencies": {
-    "@types/node": "^18",
     "dotenv": "^16.4.2",
     "esbuild": "^0.19.11",
-    "tsx": "^4.7.0",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.0"
   }
 }

apps/coordinator/src/checkpointer.ts

@@ -193,7 +193,7 @@ export class Checkpointer {
     const start = performance.now();
     this.#logger.log(`checkpointAndPush() start`, { start, opts });
 
-    let interval: NodeJS.Timer | undefined;
+    let interval: NodeJS.Timeout | undefined;
 
     if (opts.shouldHeartbeat) {
       interval = setInterval(() => {

apps/coordinator/tsconfig.json

@@ -1,8 +1,6 @@
 {
-  "include": ["./src/**/*.ts"],
-  "exclude": ["node_modules"],
   "compilerOptions": {
-    "target": "es2016",
+    "target": "es2018",
     "module": "commonjs",
     "esModuleInterop": true,
     "resolveJsonModule": true,

apps/docker-provider/package.json

@@ -20,10 +20,8 @@
     "execa": "^8.0.1"
   },
   "devDependencies": {
-    "@types/node": "^18.19.8",
     "dotenv": "^16.4.2",
     "esbuild": "^0.19.11",
-    "tsx": "^4.7.0",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.0"
   }
 }

apps/kubernetes-provider/package.json

@@ -23,7 +23,6 @@
   "devDependencies": {
     "dotenv": "^16.4.2",
     "esbuild": "^0.19.11",
-    "tsx": "^4.7.0",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.0"
   }
 }

apps/kubernetes-provider/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es2016",
+    "target": "es2018",
     "module": "commonjs",
     "esModuleInterop": true,
     "forceConsistentCasingInFileNames": true,

apps/proxy/package.json

@@ -9,7 +9,6 @@
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^4.20240512.0",
-    "typescript": "^5.0.4",
     "wrangler": "^3.57.1"
   },
   "dependencies": {

apps/webapp/app/assets/icons/TaskCachedIcon.tsx

@@ -0,0 +1,49 @@
+export function TaskCachedIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="16"
+      height="16"
+      viewBox="0 0 16 16"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <g clipPath="url(#clip0_15584_76102)">
+        <path
+          d="M0.5 3.5L0.5 2.5C0.5 1.39543 1.39543 0.5 2.5 0.5H3.5"
+          stroke="#3B82F6"
+          strokeLinecap="square"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M15.5 12.5L15.5 13.5C15.5 14.6046 14.6046 15.5 13.5 15.5L12.5 15.5"
+          stroke="#3B82F6"
+          strokeLinecap="square"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M12.5 0.5L13.5 0.5C14.6046 0.5 15.5 1.39543 15.5 2.5L15.5 3.5"
+          stroke="#3B82F6"
+          strokeLinecap="square"
+          strokeLinejoin="round"
+        />
+        <path
+          d="M3.5 15.5L2.5 15.5C1.39543 15.5 0.5 14.6046 0.5 13.5L0.5 12.5"
+          stroke="#3B82F6"
+          strokeLinecap="square"
+          strokeLinejoin="round"
+        />
+        <path d="M11.1799 4.19V5.598H8.8479V12H7.1649V5.598H4.8219V4.19H11.1799Z" fill="#3B82F6" />
+        <line x1="6" y1="15.5" x2="10" y2="15.5" stroke="#3B82F6" />
+        <line x1="6" y1="0.5" x2="10" y2="0.5" stroke="#3B82F6" />
+        <line x1="15.5" y1="6" x2="15.5" y2="10" stroke="#3B82F6" />
+        <line x1="0.5" y1="6" x2="0.5" y2="10" stroke="#3B82F6" />
+      </g>
+      <defs>
+        <clipPath id="clip0_15584_76102">
+          <rect width="16" height="16" fill="white" />
+        </clipPath>
+      </defs>
+    </svg>
+  );
+}

apps/webapp/app/components/primitives/Switch.tsx

@@ -3,6 +3,7 @@
 import * as React from "react";
 import * as SwitchPrimitives from "@radix-ui/react-switch";
 import { cn } from "~/utils/cn";
+import { ShortcutDefinition, useShortcutKeys } from "~/hooks/useShortcutKeys";
 
 const variations = {
   large: {
@@ -23,14 +24,34 @@ const variations = {
 type SwitchProps = React.ComponentPropsWithoutRef<typeof SwitchPrimitives.Root> & {
   label?: React.ReactNode;
   variant: keyof typeof variations;
+  shortcut?: ShortcutDefinition;
 };
 
 export const Switch = React.forwardRef<React.ElementRef<typeof SwitchPrimitives.Root>, SwitchProps>(
   ({ className, variant, label, ...props }, ref) => {
+    const innerRef = React.useRef<HTMLButtonElement>(null);
+    React.useImperativeHandle(ref, () => innerRef.current as HTMLButtonElement);
+
     const { container, root, thumb, text } = variations[variant];
 
+    if (props.shortcut) {
+      useShortcutKeys({
+        shortcut: props.shortcut,
+        action: () => {
+          if (innerRef.current) {
+            innerRef.current.click();
+          }
+        },
+        disabled: props.disabled,
+      });
+    }
+
     return (
-      <SwitchPrimitives.Root className={cn("group", container, className)} {...props} ref={ref}>
+      <SwitchPrimitives.Root
+        className={cn("group", container, className)}
+        {...props}
+        ref={innerRef}
+      >
         {label ? (
           <label className={cn("whitespace-nowrap", text)}>
             {typeof label === "string" ? <span>{label}</span> : label}

apps/webapp/app/components/runs/v3/BatchFilters.tsx

@@ -359,8 +359,8 @@ function BatchIdDropdown({
   if (batchId) {
     if (!batchId.startsWith("batch_")) {
       error = "Batch IDs start with 'batch_'";
-    } else if (batchId.length !== 27) {
-      error = "Batch IDs are 27 characters long";
+    } else if (batchId.length !== 27 && batchId.length !== 31) {
+      error = "Batch IDs are 27/32 characters long";
     }
   }
 

apps/webapp/app/components/runs/v3/RunFilters.tsx

@@ -763,8 +763,8 @@ function RunIdDropdown({
   if (runId) {
     if (!runId.startsWith("run_")) {
       error = "Run IDs start with 'run_'";
-    } else if (runId.length !== 25) {
-      error = "Run IDs are 25 characters long";
+    } else if (runId.length !== 25 && runId.length !== 29) {
+      error = "Run IDs are 25/30 characters long";
     }
   }
 

apps/webapp/app/components/runs/v3/RunIcon.tsx

@@ -7,6 +7,7 @@ import {
 } from "@heroicons/react/20/solid";
 import { AttemptIcon } from "~/assets/icons/AttemptIcon";
 import { TaskIcon } from "~/assets/icons/TaskIcon";
+import { TaskCachedIcon } from "~/assets/icons/TaskCachedIcon";
 import { NamedIcon } from "~/components/primitives/NamedIcon";
 import { cn } from "~/utils/cn";
 
@@ -41,6 +42,8 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) {
   switch (name) {
     case "task":
       return <TaskIcon className={cn(className, "text-blue-500")} />;
+    case "task-cached":
+      return <TaskCachedIcon className={cn(className, "text-blue-500")} />;
     case "scheduled":
       return <ClockIcon className={cn(className, "text-sun-500")} />;
     case "attempt":

apps/webapp/app/consts.ts

@@ -1,6 +1,5 @@
 export const LIVE_ENVIRONMENT = "live";
 export const DEV_ENVIRONMENT = "development";
-export const CURRENT_DEPLOYMENT_LABEL = "current";
 export const MAX_LIVE_PROJECTS = 1;
 export const DEFAULT_MAX_CONCURRENT_RUNS = 10;
 export const MAX_CONCURRENT_RUNS_LIMIT = 20;

apps/webapp/app/entry.server.tsx

@@ -205,6 +205,8 @@ process.on("uncaughtException", (error, origin) => {
 
 const sqsEventConsumer = singleton("sqsEventConsumer", getSharedSqsEventConsumer);
 
+singleton("RunEngineEventBusHandlers", registerRunEngineEventBusHandlers);
+
 export { apiRateLimiter } from "./services/apiRateLimit.server";
 export { socketIo } from "./v3/handleSocketIo.server";
 export { wss } from "./v3/handleWebsockets.server";
@@ -214,6 +216,7 @@ import { eventLoopMonitor } from "./eventLoopMonitor.server";
 import { env } from "./env.server";
 import { logger } from "./services/logger.server";
 import { Prisma } from "./db.server";
+import { registerRunEngineEventBusHandlers } from "./v3/runEngineHandlers.server";
 
 if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
   eventLoopMonitor.enable();

apps/webapp/app/env.server.ts

@@ -91,6 +91,33 @@ const EnvironmentSchema = z.object({
   REDIS_PASSWORD: z.string().optional(),
   REDIS_TLS_DISABLED: z.string().optional(),
 
+  // Valkey options (used in Run Engine 2.0+)
+  VALKEY_HOST: z
+    .string()
+    .nullish()
+    .default(process.env.REDIS_HOST ?? null),
+  VALKEY_READER_HOST: z
+    .string()
+    .nullish()
+    .default(process.env.REDIS_READER_HOST ?? null),
+  VALKEY_READER_PORT: z.coerce
+    .number()
+    .nullish()
+    .default(process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : null),
+  VALKEY_PORT: z.coerce
+    .number()
+    .nullish()
+    .default(process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : null),
+  VALKEY_USERNAME: z
+    .string()
+    .nullish()
+    .default(process.env.REDIS_USERNAME ?? null),
+  VALKEY_PASSWORD: z
+    .string()
+    .nullish()
+    .default(process.env.REDIS_PASSWORD ?? null),
+  VALKEY_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"),
+
   DEFAULT_ENV_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_ORG_EXECUTION_CONCURRENCY_LIMIT: z.coerce.number().int().default(10),
   DEFAULT_DEV_ENV_EXECUTION_ATTEMPTS: z.coerce.number().int().positive().default(1),
@@ -162,6 +189,7 @@ const EnvironmentSchema = z.object({
   SHARED_QUEUE_CONSUMER_POOL_SIZE: z.coerce.number().int().default(10),
   SHARED_QUEUE_CONSUMER_INTERVAL_MS: z.coerce.number().int().default(100),
   SHARED_QUEUE_CONSUMER_NEXT_TICK_INTERVAL_MS: z.coerce.number().int().default(100),
+  MANAGED_WORKER_SECRET: z.string().default("managed-secret"),
 
   // Development OTEL environment variables
   DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(),
@@ -261,6 +289,27 @@ const EnvironmentSchema = z.object({
   BATCH_METADATA_OPERATIONS_FLUSH_INTERVAL_MS: z.coerce.number().int().default(1000),
   BATCH_METADATA_OPERATIONS_FLUSH_ENABLED: z.string().default("1"),
   BATCH_METADATA_OPERATIONS_FLUSH_LOGGING_ENABLED: z.string().default("1"),
+
+  // Run Engine 2.0
+  RUN_ENGINE_WORKER_COUNT: z.coerce.number().int().default(4),
+  RUN_ENGINE_TASKS_PER_WORKER: z.coerce.number().int().default(10),
+  RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(100),
+  RUN_ENGINE_TIMEOUT_PENDING_EXECUTING: z.coerce.number().int().default(60_000),
+  RUN_ENGINE_TIMEOUT_PENDING_CANCEL: z.coerce.number().int().default(60_000),
+  RUN_ENGINE_TIMEOUT_EXECUTING: z.coerce.number().int().default(60_000),
+  RUN_ENGINE_TIMEOUT_EXECUTING_WITH_WAITPOINTS: z.coerce.number().int().default(60_000),
+  RUN_ENGINE_DEBUG_WORKER_NOTIFICATIONS: z.coerce.boolean().default(false),
+
+  /** How long should the presence ttl last */
+  DEV_PRESENCE_TTL_MS: z.coerce.number().int().default(30_000),
+  DEV_PRESENCE_POLL_INTERVAL_MS: z.coerce.number().int().default(5_000),
+  DEV_PRESENCE_RECONNECT_THRESHOLD_MS: z.coerce.number().int().default(2_000),
+  /** How many ms to wait until dequeuing again, if there was a run last time */
+  DEV_DEQUEUE_INTERVAL_WITH_RUN: z.coerce.number().int().default(250),
+  /** How many ms to wait until dequeuing again, if there was no run last time */
+  DEV_DEQUEUE_INTERVAL_WITHOUT_RUN: z.coerce.number().int().default(1_000),
+  /** The max number of runs per API call that we'll dequeue in DEV */
+  DEV_DEQUEUE_MAX_RUNS_PER_PULL: z.coerce.number().int().default(10),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;