huge refactoring into independent packages

.github/workflows/build.yml

@@ -8,10 +8,10 @@ on:
 
 jobs:
   macos_x86_64:
-    runs-on: macos-latest
+    runs-on: macos-13
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Crystal
         run: brew update && brew install crystal || true
       - name: Copy static libraries
@@ -31,9 +31,12 @@ jobs:
           cp $(brew --prefix)/opt/pcre2/lib/libpcre2-8.a ./libs
       - name: Build the binary
         # Statically link most non-system libraries
-        run: env CRYSTAL_LIBRARY_PATH=`pwd`/libs shards build --no-debug --release -Dpreview_mt
+        run: |
+          env CRYSTAL_LIBRARY_PATH=`pwd`/libs crystal projects.cr build:cli --no-debug --release #-Dpreview_mt
+          mkdir bin
+          mv ./packages/cli/bin/zap ./bin/zap
       - name: Upload a Build Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: zap_x86_64-apple-darwin
           path: ./bin/zap
@@ -42,7 +45,7 @@ jobs:
     runs-on: macos-14
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Crystal
         run: brew update && brew install crystal || true
       - name: Copy static libraries
@@ -61,9 +64,12 @@ jobs:
           # libpcre
           cp $(brew --prefix)/opt/pcre2/lib/libpcre2-8.a ./libs
       - name: Build the binary
-        run: env CRYSTAL_LIBRARY_PATH=`pwd`/libs shards build --no-debug --release -Dpreview_mt
+        run: |
+          env CRYSTAL_LIBRARY_PATH=`pwd`/libs crystal projects.cr build:cli --no-debug --release  #-Dpreview_mt
+          mkdir bin
+          mv ./packages/cli/bin/zap ./bin/zap
       - name: Upload a Build Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: zap_arm64-apple-darwin
           path: ./bin/zap
@@ -74,11 +80,14 @@ jobs:
       image: crystallang/crystal:latest-alpine
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build the static binary
-        run: shards build --production --release --static --no-debug --stats
+        run: |
+          crystal projects.cr build:cli --production --release --static --no-debug --stats
+          mkdir bin
+          mv ./packages/cli/bin/zap ./bin/zap
       - name: Upload a Build Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: zap_x86_64-linux-musl
           path: ./bin/zap
@@ -87,16 +96,18 @@ jobs:
     runs-on: windows-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Crystal
         uses: crystal-lang/install-crystal@v1
       - name: Build the binary
         shell: bash
         run: |
-          shards build --progress --release --no-debug --stats
+          crystal projects.cr build:cli --progress --release --no-debug --stats
+          mkdir bin
+          mv ./packages/cli/bin/zap.exe ./bin/zap.exe
           ls -al ./bin
       - name: Upload a Build Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: zap_x86_64-pc-win32.exe
           path: ${{ github.workspace }}\bin\zap.exe
@@ -107,7 +118,7 @@ jobs:
     if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
       - name: Check if release is needed

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install node
         uses: actions/setup-node@v3
         with:

.github/workflows/tests.yml

@@ -12,15 +12,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Crystal
         uses: crystal-lang/install-crystal@v1
       - name: Install dependencies
-        run: shards install
+        run: crystal projects.cr install
       - name: Run tests
-        run: crystal spec
+        run: crystal projects.cr spec
       - name: Run tests (multithreaded)
-        run: crystal spec -Dpreview_mt
+        run: crystal projects.cr spec -Dpreview_mt
   launch:
     name: Build and check if the binary is working
     strategy:
@@ -29,12 +29,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Crystal
         uses: crystal-lang/install-crystal@v1
       - name: Install dependencies
-        run: shards install
+        run: crystal projects.cr install
       - name: Build
-        run: shards build
+        run: crystal projects.cr build:cli
       - name: Print version
-        run: ./bin/zap --version
+        run: ./packages/cli/bin/zap --version

.gitignore

@@ -1,6 +1,6 @@
 /docs/
-/lib/
-/bin/
+lib/
+bin/
 /.shards/
 *.dwarf
 node_modules

README.md

@@ -188,7 +188,7 @@ Benchmarks consist on installing a fresh [**create-react-app**](https://create-r
 
 **See:** [https://github.com/elbywan/zap/tree/main/bench](/bench)
 
-They are performed on my own personal laptop (macbook pro 16" 2019, 2,3 GHz Intel Core i9, 16 Go 2667 MHz DDR4) with 5G wifi and 1 Gb/s fiber.
+They are performed on my own personal laptop (Framework Laptop 13, AMD Ryzen 5 7640U, 32 GB 5600 MHz DDR5) with 5G wifi and 1 Gb/s fiber.
 
 The benchmarking tool is [**hyperfine**](https://github.com/sharkdp/hyperfine) and to make sure that the results are consistent I re-ran unfavorable results (high error delta).
 
@@ -240,11 +240,13 @@ On top of that, zap will also try to cache package manifests in order to avoid u
 
 ```bash
 git clone https://github.com/elbywan/zap
-shards install
+crystal projects.cr spec install
 # Run the specs
-crystal spec
+crystal projects.cr spec
 # Build locally (-Dpreview_mt might not work on some os/arch)
-shards build --progress -Dpreview_mt --release
+crystal projects.cr cli:build --production  --release --progress # -Dpreview_mt
+# Run the binary
+./packages/cli/bin/zap --help
 ```
 
 ## Contributing

bench/.prototools

@@ -2,3 +2,4 @@ bun = "latest"
 node = "lts"
 pnpm = "latest"
 yarn = "latest"
+python = "3.12.0"

bench/README.md

@@ -28,6 +28,12 @@ pkgx +yarnpkg.com +node +npm +pnpm +bun +python
 
 ## Dependencies
 
+### Benchmarking
+
+Benchmarking is done using [hyperfine](https://github.com/sharkdp/hyperfine).
+
+### Plotting
+
 Plotting requires python and the following dependencies:
 
 ```bash
@@ -38,7 +44,7 @@ pip install numpy matplotlib
 
 ```bash
 # Run the benchmarks
-./bench.sh
+./bench.sh # or ./bench-local.sh to build and benchmark a local version of zap
 # Plot the results
 ./plot.sh
 ```

bench/bench-local.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Build zap
+../projects.cr build:cli --production --release --progress #-Dpreview_mt
+# Run the benchmarks
+env PATH="$(pwd)/../packages/cli/bin:$PATH" ./bench.sh
+env PATH="$(pwd)/../packages/cli/bin:$PATH" ./plot.sh

bench/plot.sh

@@ -7,7 +7,7 @@ BUN_LABEL="bun v$(bun --version)"
 ZAP_LABEL="zap $(zap --version)"
 LABELS="$NPM_LABEL,$YARN_LABEL (node linker),$PNPM_LABEL,$BUN_LABEL,$ZAP_LABEL"
 
-python3 plot.py -o cold.png --labels "$LABELS" --title "Without cache, lockfile or node modules" ./react-app/cold.json
-python3 plot.py -o only-cache.png --labels "$LABELS" --title "Without lockfile or node modules" ./react-app/only-cache.json
-python3 plot.py -o without-lockfile.png --labels "$LABELS" --title "Without lockfile" ./react-app/without-lockfile.json
-python3 plot.py -o without-node-modules.png --labels "$LABELS" --title "Without node modules" ./react-app/without-node-modules.json
+python plot.py -o cold.png --labels "$LABELS" --title "Without cache, lockfile or node modules" ./react-app/cold.json
+python plot.py -o only-cache.png --labels "$LABELS" --title "Without lockfile or node modules" ./react-app/only-cache.json
+python plot.py -o without-lockfile.png --labels "$LABELS" --title "Without lockfile" ./react-app/without-lockfile.json
+python plot.py -o without-node-modules.png --labels "$LABELS" --title "Without node modules" ./react-app/without-node-modules.json

src/backend/backend.cr → packages/backend/backend.cr

@@ -1,8 +1,9 @@
 require "file_utils"
-require "../utils/concurrent/pipeline"
+require "concurrency/pipeline"
+require "extensions/dir"
 
-module Zap::Backend
-  alias Pipeline = ::Zap::Utils::Concurrent::Pipeline
+module Backend
+  alias Pipeline = Concurrency::Pipeline
 
   enum Backends
     CloneFile
@@ -12,29 +13,6 @@ module Zap::Backend
     Symlink
   end
 
-  def self.install(*, dependency : Package, target : Path | String, backend : Backends, store : Store, &on_installing) : Bool
-    case backend
-    in .clone_file?
-      {% if flag?(:darwin) %}
-        Backend::CloneFile.install(dependency, target, store: store, &on_installing)
-      {% else %}
-        raise "The clonefile backend is not supported on this platform"
-      {% end %}
-    in .copy_file?
-      {% if flag?(:darwin) %}
-        Backend::CopyFile.install(dependency, target, store: store, &on_installing)
-      {% else %}
-        raise "The copyfile backend is not supported on this platform"
-      {% end %}
-    in .hardlink?
-      Backend::Hardlink.install(dependency, target, store: store, &on_installing)
-    in .copy?
-      Backend::Copy.install(dependency, target, store: store, &on_installing)
-    in .symlink?
-      Backend::Symlink.install(dependency, target, store: store, &on_installing)
-    end
-  end
-
   # -----------------
   # Iterative version
   # -----------------
@@ -119,10 +97,64 @@ module Zap::Backend
   #   end
   # end
 
-  protected def self.prepare(dependency : Package, dest_path : Path | String, *, store : Store, mkdir_parent = false) : {Path, Path, Bool}
+  protected def self.prepare(dependency : Data::Package, dest_path : Path | String, *, store : Store) : {Path, Path, Bool}
     src_path = store.package_path(dependency)
-    already_installed = Installer.package_already_installed?(dependency, dest_path)
-    Utils::Directories.mkdir_p(mkdir_parent ? dest_path.dirname : dest_path) unless already_installed
+    already_installed = self.package_already_installed?(dependency.key, dest_path)
+    Utils::Directories.mkdir_p(dest_path.dirname) unless already_installed
     {src_path, dest_path, already_installed}
   end
+
+  # Check if a package is already installed on the filesystem
+  def self.package_already_installed?(package_key : String, path : Path)
+    if exists = Dir.exists?(path)
+      metadata_path = path / Shared::Constants::METADATA_FILE_NAME
+      unless File.readable?(metadata_path)
+        FileUtils.rm_rf(path)
+        exists = false
+      else
+        key = File.read(metadata_path)
+        if key != package_key
+          FileUtils.rm_rf(path)
+          exists = false
+        end
+      end
+    end
+    exists
+  end
+end
+
+require "./clonefile"
+require "./copy"
+require "./copyfile"
+require "./hardlink"
+require "./symlink"
+
+module Backend
+  def self.link(*, dependency : Data::Package, target : Path | String, backend : Backends, store : Store, &on_installing) : Bool
+    src_path, dest_path, already_installed = self.prepare(dependency, target, store: store)
+    return false if already_installed
+
+    yield
+
+    case backend
+    in .clone_file?
+      {% if flag?(:darwin) %}
+        Backend::CloneFile.link(src_path, dest_path)
+      {% else %}
+        raise "The clonefile backend is not supported on this platform"
+      {% end %}
+    in .copy_file?
+      {% if flag?(:darwin) %}
+        Backend::CopyFile.link(src_path, dest_path)
+      {% else %}
+        raise "The copyfile backend is not supported on this platform"
+      {% end %}
+    in .hardlink?
+      Backend::Hardlink.link(src_path, dest_path)
+    in .copy?
+      Backend::Copy.link(src_path, dest_path)
+    in .symlink?
+      Backend::Symlink.link(src_path, dest_path)
+    end
+  end
 end

packages/backend/clonefile.cr

@@ -0,0 +1,11 @@
+require "extensions/libc/clonefile"
+
+module Backend::CloneFile
+  def self.link(src_path : String | Path, dest_path : String | Path) : Bool
+    result = LibC.clonefile(src_path.to_s, dest_path.to_s, 0)
+    if result == -1
+      raise "Error cloning file: #{Errno.value} #{src_path.to_s} -> #{dest_path.to_s}"
+    end
+    true
+  end
+end

packages/backend/copy.cr

@@ -0,0 +1,13 @@
+require "concurrency/pipeline"
+
+module Backend::Copy
+  def self.link(src_path : String | Path, dest_path : String | Path) : Bool
+    # FileUtils.cp_r(src_path, dest_path)
+    Pipeline.wrap(force_wait: true) do |pipeline|
+      Backend.recursively(src_path.to_s, dest_path.to_s, pipeline: pipeline) do |src, dest|
+        File.copy(src, dest)
+      end
+    end
+    true
+  end
+end

packages/backend/copyfile.cr

@@ -0,0 +1,13 @@
+require "extensions/libc/copyfile"
+require "concurrency/pipeline"
+
+module Backend::CopyFile
+  def self.link(src_path : String | Path, dest_path : String | Path) : Bool
+    Pipeline.wrap(force_wait: true) do |pipeline|
+      Backend.recursively(src_path.to_s, dest_path.to_s, pipeline) do |src, dest|
+        LibC.copyfile(src.to_s, dest.to_s, nil, LibC::COPYFILE_CLONE_FORCE | LibC::COPYFILE_ALL)
+      end
+    end
+    true
+  end
+end

packages/backend/hardlink.cr

@@ -0,0 +1,12 @@
+require "concurrency/pipeline"
+
+module Backend::Hardlink
+  def self.link(src_path : String | Path, dest_path : String | Path) : Bool
+    Pipeline.wrap(force_wait: true) do |pipeline|
+      Backend.recursively(src_path.to_s, dest_path.to_s, pipeline: pipeline) do |src, dest|
+        File.link(src, dest)
+      end
+    end
+    true
+  end
+end