Add S3 file storage

packages/file-storage/package.json

@@ -17,6 +17,7 @@
   "type": "module",
   "exports": {
     ".": "./dist/file-storage.js",
+    "./aws": "./dist/aws.js",
     "./local": "./dist/local.js",
     "./memory": "./dist/memory.js",
     "./package.json": "./package.json"
@@ -25,17 +26,22 @@
     "@mjackson/lazy-file": "^3.1.0"
   },
   "devDependencies": {
-    "@types/node": "^20.14.10"
+    "@types/node": "^20.14.10",
+    "vitest": "^2.0.5"
   },
   "scripts": {
     "build": "tsc --outDir dist --project tsconfig.lib.json",
-    "test": "node --import @swc-node/register/esm-register --test ./src/**/*.spec.ts",
+    "test": "vitest run",
     "prepare": "pnpm run build"
   },
   "keywords": [
     "file",
     "storage",
     "stream",
     "fs"
-  ]
+  ],
+  "optionalDependencies": {
+    "@aws-sdk/client-s3": "^3.645.0",
+    "@aws-sdk/lib-storage": "^3.645.0"
+  }
 }

packages/file-storage/src/aws.ts

@@ -0,0 +1 @@
+export { S3FileStorage } from "./lib/s3-file-storage.js";

packages/file-storage/src/lib/local-file-storage.spec.ts

@@ -1,5 +1,5 @@
 import * as assert from 'node:assert/strict';
-import { after, describe, it } from 'node:test';
+import { afterAll, describe, it } from "vitest";
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
@@ -10,7 +10,7 @@ const __dirname = new URL('.', import.meta.url).pathname;
 describe('LocalFileStorage', () => {
   let directory = path.resolve(__dirname, '../../test-local-file-storage');
 
-  after(() => {
+  afterAll(() => {
     fs.rmSync(directory, { recursive: true });
   });
 

packages/file-storage/src/lib/memory-file-storage.spec.ts

@@ -1,5 +1,5 @@
 import * as assert from 'node:assert/strict';
-import { describe, it } from 'node:test';
+import { describe, it } from "vitest";
 
 import { MemoryFileStorage } from './memory-file-storage.js';
 

packages/file-storage/src/lib/s3-file-storage.spec.ts

@@ -0,0 +1,102 @@
+import { vi, expect, describe, afterEach, beforeEach, it } from "vitest";
+import {
+  DeleteObjectCommand,
+  HeadObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { S3FileStorage } from "./s3-file-storage.js";
+import { LazyFile } from "@mjackson/lazy-file";
+
+vi.mock("@aws-sdk/client-s3", () => {
+  return {
+    S3Client: vi.fn().mockImplementation(() => ({
+      send: vi.fn(),
+    })),
+    HeadObjectCommand: vi.fn(),
+    GetObjectCommand: vi.fn(),
+    DeleteObjectCommand: vi.fn(),
+  };
+});
+
+vi.mock("@aws-sdk/lib-storage", () => {
+  return {
+    Upload: vi.fn().mockReturnValue({
+      done: vi.fn(),
+    }),
+  };
+});
+
+describe("S3FileStorage", () => {
+  let s3Client: S3Client;
+  let s3FileStorage: S3FileStorage;
+
+  beforeEach(() => {
+    s3Client = new S3Client();
+    s3FileStorage = new S3FileStorage(s3Client, "test-bucket");
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("checks if a file exists", async () => {
+    const result = await s3FileStorage.has("existing-key");
+    expect(result).toBe(true);
+    expect(HeadObjectCommand).toHaveBeenCalledWith({
+      Bucket: "test-bucket",
+      Key: "existing-key",
+    });
+    expect(vi.mocked(s3Client.send).mock.calls[0][0]).toBeInstanceOf(
+      HeadObjectCommand,
+    );
+  });
+
+  it("stores a file", async () => {
+    const file = new File(["content"], "file.txt");
+    await s3FileStorage.set("file-key", file);
+
+    expect(Upload).toHaveBeenCalledWith({
+      client: s3Client,
+      params: {
+        Bucket: "test-bucket",
+        Key: "file-key",
+        Body: expect.any(ReadableStream),
+      },
+    });
+    // @ts-ignore
+    const uploadInstance = Upload.mock.results[0].value;
+    expect(uploadInstance.done).toHaveBeenCalled();
+  });
+
+  it("retrieves a file", async () => {
+    // @ts-ignore
+    vi.mocked(s3Client.send).mockResolvedValueOnce({
+      ContentLength: 123,
+      ContentType: "text/plain",
+    });
+
+    const result = await s3FileStorage.get("existing-key");
+
+    expect(HeadObjectCommand).toHaveBeenCalledWith({
+      Bucket: "test-bucket",
+      Key: "existing-key",
+    });
+
+    expect(result).toBeInstanceOf(LazyFile);
+    expect(vi.mocked(s3Client.send).mock.calls[0][0]).toBeInstanceOf(
+      HeadObjectCommand,
+    );
+  });
+
+  it("removes a file", async () => {
+    await s3FileStorage.remove("existing-key");
+    expect(DeleteObjectCommand).toHaveBeenCalledWith({
+      Bucket: "test-bucket",
+      Key: "existing-key",
+    });
+    expect(vi.mocked(s3Client.send).mock.calls[0][0]).toBeInstanceOf(
+      DeleteObjectCommand,
+    );
+  });
+});

packages/file-storage/src/lib/s3-file-storage.ts

@@ -0,0 +1,123 @@
+import type { S3Client } from "@aws-sdk/client-s3";
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  HeadObjectCommand,
+} from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { LazyFile, LazyContent } from "@mjackson/lazy-file";
+import { FileStorage } from "./file-storage.js";
+
+/**
+ * An S3 implementation of the `FileStorage` interface.
+ */
+export class S3FileStorage implements FileStorage {
+  #s3: S3Client;
+  #bucketName: string;
+
+  constructor(s3: S3Client, bucketName: string) {
+    this.#s3 = s3;
+    this.#bucketName = bucketName;
+  }
+
+  async has(key: string): Promise<boolean> {
+    try {
+      await this.#s3.send(
+        new HeadObjectCommand({
+          Bucket: this.#bucketName,
+          Key: key,
+        }),
+      );
+      return true;
+    } catch (error) {
+      if (error instanceof Error ? error.name === "NotFound" : "") {
+        return false;
+      }
+      throw error;
+    }
+  }
+
+  async set(key: string, file: File): Promise<void> {
+    const upload = new Upload({
+      client: this.#s3,
+      params: {
+        Bucket: this.#bucketName,
+        Key: key,
+        Body: file.stream(),
+      },
+    });
+    await upload.done();
+  }
+
+  async get(key: string): Promise<LazyFile | null> {
+    try {
+      const head = await this.#s3.send(
+        new HeadObjectCommand({
+          Bucket: this.#bucketName,
+          Key: key,
+        }),
+      );
+
+      const contentLength = Number(head.ContentLength);
+      const contentType = head.ContentType || "application/octet-stream";
+
+      const s3 = this.#s3;
+      const bucketName = this.#bucketName;
+
+      const lazyContent: LazyContent = {
+        byteLength: contentLength,
+        stream: (start: number = 0, end: number = contentLength) => {
+          const range = `bytes=${start}-${end - 1}`;
+          return new ReadableStream({
+            async start(controller) {
+              try {
+                const command = new GetObjectCommand({
+                  Bucket: bucketName,
+                  Key: key,
+                  Range: range,
+                });
+                const { Body } = await s3.send(command);
+
+                if (!Body) {
+                  throw new Error("Failed to retrieve a valid stream from S3");
+                }
+
+                const reader = Body.transformToWebStream().getReader();
+
+                const read = async () => {
+                  const { done, value } = await reader.read();
+                  if (done) {
+                    controller.close();
+                  } else {
+                    controller.enqueue(value);
+                    await read();
+                  }
+                };
+
+                await read();
+              } catch (error) {
+                controller.error(error);
+              }
+            },
+          });
+        },
+      };
+
+      return new LazyFile(lazyContent, key, { type: contentType });
+    } catch (error) {
+      if (error instanceof Error && error.name === "NoSuchKey") {
+        return null;
+      }
+      throw error;
+    }
+  }
+
+  async remove(key: string): Promise<void> {
+    await this.#s3.send(
+      new DeleteObjectCommand({
+        Bucket: this.#bucketName,
+        Key: key,
+      }),
+    );
+  }
+}