feat(build): Add support for Python scripts via pythonExtension

.changeset/little-trains-begin.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/build": minor
+---
+
+Introduced a new Python extension to enhance the build process. It now allows users to execute Python scripts with improved support and error handling.

packages/build/package.json

@@ -28,6 +28,7 @@
       "./extensions/prisma": "./src/extensions/prisma.ts",
       "./extensions/audioWaveform": "./src/extensions/audioWaveform.ts",
       "./extensions/typescript": "./src/extensions/typescript.ts",
+      "./extensions/python": "./src/extensions/python.ts",
       "./extensions/puppeteer": "./src/extensions/puppeteer.ts"
     },
     "sourceDialects": [
@@ -51,6 +52,9 @@
       "extensions/typescript": [
         "dist/commonjs/extensions/typescript.d.ts"
       ],
+      "extensions/python": [
+        "dist/commonjs/extensions/python.d.ts"
+      ],
       "extensions/puppeteer": [
         "dist/commonjs/extensions/puppeteer.d.ts"
       ]
@@ -66,7 +70,9 @@
   },
   "dependencies": {
     "@trigger.dev/core": "workspace:3.3.14",
+    "@trigger.dev/sdk": "workspace:3.3.14",
     "pkg-types": "^1.1.3",
+    "tinyexec": "^0.3.2",
     "tinyglobby": "^0.2.2",
     "tsconfck": "3.1.3"
   },
@@ -150,6 +156,17 @@
         "default": "./dist/commonjs/extensions/typescript.js"
       }
     },
+    "./extensions/python": {
+      "import": {
+        "@triggerdotdev/source": "./src/extensions/python.ts",
+        "types": "./dist/esm/extensions/python.d.ts",
+        "default": "./dist/esm/extensions/python.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/extensions/python.d.ts",
+        "default": "./dist/commonjs/extensions/python.js"
+      }
+    },
     "./extensions/puppeteer": {
       "import": {
         "@triggerdotdev/source": "./src/extensions/puppeteer.ts",

packages/build/src/extensions/python.ts

@@ -0,0 +1,175 @@
+import fs from "node:fs";
+import assert from "node:assert";
+import { additionalFiles } from "./core/additionalFiles.js";
+import { BuildManifest } from "@trigger.dev/core/v3";
+import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build";
+import { logger } from "@trigger.dev/sdk/v3";
+import { x, Options as XOptions, Result } from "tinyexec";
+
+export type PythonOptions = {
+  requirements?: string[];
+  requirementsFile?: string;
+  /**
+   * [Dev-only] The path to the python binary.
+   *
+   * @remarks
+   * This option is typically used during local development or in specific testing environments
+   * where a particular Python installation needs to be targeted.  It should point to the full path of the python executable.
+   *
+   * Example: `/usr/bin/python3` or `C:\\Python39\\python.exe`
+   */
+  pythonBinaryPath?: string;
+  /**
+   * An array of glob patterns that specify which Python scripts are allowed to be executed.
+   *
+   * @remarks
+   * These scripts will be copied to the container during the build process.
+   */
+  scripts?: string[];
+};
+
+const splitAndCleanComments = (str: string) =>
+  str
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line && !line.startsWith("#"));
+
+export function pythonExtension(options: PythonOptions = {}): BuildExtension {
+  return new PythonExtension(options);
+}
+
+class PythonExtension implements BuildExtension {
+  public readonly name = "PythonExtension";
+
+  constructor(private options: PythonOptions = {}) {
+    assert(
+      !(this.options.requirements && this.options.requirementsFile),
+      "Cannot specify both requirements and requirementsFile"
+    );
+
+    if (this.options.requirementsFile) {
+      assert(
+        fs.existsSync(this.options.requirementsFile),
+        `Requirements file not found: ${this.options.requirementsFile}`
+      );
+      this.options.requirements = splitAndCleanComments(
+        fs.readFileSync(this.options.requirementsFile, "utf-8")
+      );
+    }
+  }
+
+  async onBuildComplete(context: BuildContext, manifest: BuildManifest) {
+    await additionalFiles({
+      files: this.options.scripts ?? [],
+    }).onBuildComplete!(context, manifest);
+
+    if (context.target === "dev") {
+      if (this.options.pythonBinaryPath) {
+        process.env.PYTHON_BIN_PATH = this.options.pythonBinaryPath;
+      }
+
+      return;
+    }
+
+    context.logger.debug(`Adding ${this.name} to the build`);
+
+    context.addLayer({
+      id: "python-installation",
+      image: {
+        instructions: splitAndCleanComments(`
+          # Install Python
+          RUN apt-get update && apt-get install -y --no-install-recommends \
+              python3 python3-pip python3-venv && \
+              apt-get clean && rm -rf /var/lib/apt/lists/*
+
+          # Set up Python environment
+          RUN python3 -m venv /opt/venv
+          ENV PATH="/opt/venv/bin:$PATH"
+        `),
+      },
+      deploy: {
+        env: {
+          PYTHON_BIN_PATH: `/opt/venv/bin/python`,
+        },
+        override: true,
+      },
+    });
+
+    context.addLayer({
+      id: "python-dependencies",
+      build: {
+        env: {
+          REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "",
+        },
+      },
+      image: {
+        instructions: splitAndCleanComments(`
+          ARG REQUIREMENTS_CONTENT
+          RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt
+
+          # Install dependencies
+          RUN pip install --no-cache-dir -r requirements.txt
+        `),
+      },
+      deploy: {
+        override: true,
+      },
+    });
+  }
+}
+
+export const run = async (
+  scriptArgs: string[] = [],
+  options: Partial<XOptions> = {}
+): Promise<Result> => {
+  const pythonBin = process.env.PYTHON_BIN_PATH || "python";
+
+  const result = await x(pythonBin, scriptArgs, {
+    ...options,
+    throwOnError: false, // Ensure errors are handled manually
+  });
+
+  try {
+    assert(
+      result.exitCode === 0,
+      `Python command exited with non-zero code ${result.exitCode}\nStdout: ${result.stdout}\nStderr: ${result.stderr}`
+    );
+  } catch (error) {
+    logger.error("Python command execution failed", {
+      error: error instanceof Error ? error.message : error,
+      command: `${pythonBin} ${scriptArgs.join(" ")}`,
+      stdout: result.stdout,
+      stderr: result.stderr,
+      exitCode: result.exitCode,
+    });
+    throw error;
+  }
+
+  return result;
+};
+
+export const runScript = (
+  scriptPath: string,
+  scriptArgs: string[] = [],
+  options: Partial<XOptions> = {}
+) => {
+  assert(scriptPath, "Script path is required");
+  assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`);
+
+  return run([scriptPath, ...scriptArgs], options);
+};
+
+export const runInline = async (scriptContent: string, options: Partial<XOptions> = {}) => {
+  assert(scriptContent, "Script content is required");
+
+  const tmpFile = `/tmp/script_${Date.now()}.py`;
+  await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 });
+
+  try {
+    return await runScript(tmpFile, [], options);
+  } finally {
+    await fs.promises.unlink(tmpFile);
+  }
+};
+
+export default { run, runScript, runInline };

references/v3-catalog/trigger.config.ts

@@ -7,6 +7,7 @@ import { ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core";
 import { puppeteer } from "@trigger.dev/build/extensions/puppeteer";
 import { prismaExtension } from "@trigger.dev/build/extensions/prisma";
 import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript";
+import { pythonExtension } from "@trigger.dev/build/extensions/python";
 import { defineConfig } from "@trigger.dev/sdk/v3";
 
 export { handleError } from "./src/handleError.js";
@@ -86,6 +87,7 @@ export default defineConfig({
           value: secret.secretValue,
         }));
       }),
+      pythonExtension(),
       puppeteer(),
     ],
     external: ["re2"],