diff --git a/.crystalline_main.cr b/.crystalline_main.cr deleted file mode 100644 index 5b00f6c..0000000 --- a/.crystalline_main.cr +++ /dev/null @@ -1,2 +0,0 @@ -# require "./spec/**" -require "./src/cli/cli.cr" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32527c6..ad1f981 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,10 +8,10 @@ on: jobs: macos_x86_64: - runs-on: macos-latest + runs-on: macos-13 steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Crystal run: brew update && brew install crystal || true - name: Copy static libraries @@ -31,9 +31,12 @@ jobs: cp $(brew --prefix)/opt/pcre2/lib/libpcre2-8.a ./libs - name: Build the binary # Statically link most non-system libraries - run: env CRYSTAL_LIBRARY_PATH=`pwd`/libs shards build --no-debug --release -Dpreview_mt + run: | + env CRYSTAL_LIBRARY_PATH=`pwd`/libs crystal projects.cr build:cli --no-debug --release #-Dpreview_mt + mkdir bin + mv ./packages/cli/bin/zap ./bin/zap - name: Upload a Build Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: zap_x86_64-apple-darwin path: ./bin/zap @@ -42,7 +45,7 @@ jobs: runs-on: macos-14 steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Crystal run: brew update && brew install crystal || true - name: Copy static libraries @@ -61,9 +64,12 @@ jobs: # libpcre cp $(brew --prefix)/opt/pcre2/lib/libpcre2-8.a ./libs - name: Build the binary - run: env CRYSTAL_LIBRARY_PATH=`pwd`/libs shards build --no-debug --release -Dpreview_mt + run: | + env CRYSTAL_LIBRARY_PATH=`pwd`/libs crystal projects.cr build:cli --no-debug --release #-Dpreview_mt + mkdir bin + mv ./packages/cli/bin/zap ./bin/zap - name: Upload a Build Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: zap_arm64-apple-darwin path: ./bin/zap @@ -74,11 +80,14 @@ jobs: image: crystallang/crystal:latest-alpine steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build the static binary - run: shards build --production --release --static --no-debug --stats + run: | + crystal projects.cr build:cli --production --release --static --no-debug --stats + mkdir bin + mv ./packages/cli/bin/zap ./bin/zap - name: Upload a Build Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: zap_x86_64-linux-musl path: ./bin/zap @@ -87,16 +96,18 @@ jobs: runs-on: windows-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Crystal uses: crystal-lang/install-crystal@v1 - name: Build the binary shell: bash run: | - shards build --progress --release --no-debug --stats + crystal projects.cr build:cli --progress --release --no-debug --stats + mkdir bin + mv ./packages/cli/bin/zap.exe ./bin/zap.exe ls -al ./bin - name: Upload a Build Artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: zap_x86_64-pc-win32.exe path: ${{ github.workspace }}\bin\zap.exe @@ -107,7 +118,7 @@ jobs: if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 2 - name: Check if release is needed diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e60ea7..c56a0ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install node uses: actions/setup-node@v3 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f06c8d2..70b503a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,15 +12,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Crystal uses: crystal-lang/install-crystal@v1 - name: Install dependencies - run: shards install + run: crystal projects.cr install - name: Run tests - run: crystal spec + run: crystal projects.cr spec - name: Run tests (multithreaded) - run: crystal spec -Dpreview_mt + run: crystal projects.cr spec -Dpreview_mt launch: name: Build and check if the binary is working strategy: @@ -29,12 +29,12 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install Crystal uses: crystal-lang/install-crystal@v1 - name: Install dependencies - run: shards install + run: crystal projects.cr install - name: Build - run: shards build + run: crystal projects.cr build:cli - name: Print version - run: ./bin/zap --version + run: ./packages/cli/bin/zap --version diff --git a/.gitignore b/.gitignore index ea067ad..986d03c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /docs/ -/lib/ -/bin/ +lib/ +bin/ /.shards/ *.dwarf node_modules diff --git a/README.md b/README.md index 245df26..aa7fb34 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ Benchmarks consist on installing a fresh [**create-react-app**](https://create-r **See:** [https://github.com/elbywan/zap/tree/main/bench](/bench) -They are performed on my own personal laptop (macbook pro 16" 2019, 2,3 GHz Intel Core i9, 16 Go 2667 MHz DDR4) with 5G wifi and 1 Gb/s fiber. +They are performed on my own personal laptop (Framework Laptop 13, AMD Ryzen 5 7640U, 32 GB 5600 MHz DDR5) with 5G wifi and 1 Gb/s fiber. The benchmarking tool is [**hyperfine**](https://github.com/sharkdp/hyperfine) and to make sure that the results are consistent I re-ran unfavorable results (high error delta). @@ -240,11 +240,13 @@ On top of that, zap will also try to cache package manifests in order to avoid u ```bash git clone https://github.com/elbywan/zap -shards install +crystal projects.cr spec install # Run the specs -crystal spec +crystal projects.cr spec # Build locally (-Dpreview_mt might not work on some os/arch) -shards build --progress -Dpreview_mt --release +crystal projects.cr cli:build --production --release --progress # -Dpreview_mt +# Run the binary +./packages/cli/bin/zap --help ``` ## Contributing diff --git a/bench/.prototools b/bench/.prototools index a1a0bd9..f6c0637 100644 --- a/bench/.prototools +++ b/bench/.prototools @@ -2,3 +2,4 @@ bun = "latest" node = "lts" pnpm = "latest" yarn = "latest" +python = "3.12.0" diff --git a/bench/README.md b/bench/README.md index d313bb3..4038a63 100644 --- a/bench/README.md +++ b/bench/README.md @@ -28,6 +28,12 @@ pkgx +yarnpkg.com +node +npm +pnpm +bun +python ## Dependencies +### Benchmarking + +Benchmarking is done using [hyperfine](https://github.com/sharkdp/hyperfine). + +### Plotting + Plotting requires python and the following dependencies: ```bash @@ -38,7 +44,7 @@ pip install numpy matplotlib ```bash # Run the benchmarks -./bench.sh +./bench.sh # or ./bench-local.sh to build and benchmark a local version of zap # Plot the results ./plot.sh ``` \ No newline at end of file diff --git a/bench/bench-local.sh b/bench/bench-local.sh new file mode 100755 index 0000000..b4563ac --- /dev/null +++ b/bench/bench-local.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Build zap +../projects.cr build:cli --production --release --progress #-Dpreview_mt +# Run the benchmarks +env PATH="$(pwd)/../packages/cli/bin:$PATH" ./bench.sh +env PATH="$(pwd)/../packages/cli/bin:$PATH" ./plot.sh \ No newline at end of file diff --git a/bench/cold.png b/bench/cold.png index 5d41b31..d8a95ee 100644 Binary files a/bench/cold.png and b/bench/cold.png differ diff --git a/bench/only-cache.png b/bench/only-cache.png index 94ca746..7afcbdf 100644 Binary files a/bench/only-cache.png and b/bench/only-cache.png differ diff --git a/bench/plot.sh b/bench/plot.sh index 8c63409..7b2d79c 100755 --- a/bench/plot.sh +++ b/bench/plot.sh @@ -7,7 +7,7 @@ BUN_LABEL="bun v$(bun --version)" ZAP_LABEL="zap $(zap --version)" LABELS="$NPM_LABEL,$YARN_LABEL (node linker),$PNPM_LABEL,$BUN_LABEL,$ZAP_LABEL" -python3 plot.py -o cold.png --labels "$LABELS" --title "Without cache, lockfile or node modules" ./react-app/cold.json -python3 plot.py -o only-cache.png --labels "$LABELS" --title "Without lockfile or node modules" ./react-app/only-cache.json -python3 plot.py -o without-lockfile.png --labels "$LABELS" --title "Without lockfile" ./react-app/without-lockfile.json -python3 plot.py -o without-node-modules.png --labels "$LABELS" --title "Without node modules" ./react-app/without-node-modules.json \ No newline at end of file +python plot.py -o cold.png --labels "$LABELS" --title "Without cache, lockfile or node modules" ./react-app/cold.json +python plot.py -o only-cache.png --labels "$LABELS" --title "Without lockfile or node modules" ./react-app/only-cache.json +python plot.py -o without-lockfile.png --labels "$LABELS" --title "Without lockfile" ./react-app/without-lockfile.json +python plot.py -o without-node-modules.png --labels "$LABELS" --title "Without node modules" ./react-app/without-node-modules.json \ No newline at end of file diff --git a/bench/without-lockfile.png b/bench/without-lockfile.png index 11b81f0..7dfa89d 100644 Binary files a/bench/without-lockfile.png and b/bench/without-lockfile.png differ diff --git a/bench/without-node-modules.png b/bench/without-node-modules.png index 67887b7..9accff3 100644 Binary files a/bench/without-node-modules.png and b/bench/without-node-modules.png differ diff --git a/src/backend/backend.cr b/packages/backend/backend.cr similarity index 72% rename from src/backend/backend.cr rename to packages/backend/backend.cr index 477bfc0..3f4e37b 100644 --- a/src/backend/backend.cr +++ b/packages/backend/backend.cr @@ -1,8 +1,9 @@ require "file_utils" -require "../utils/concurrent/pipeline" +require "concurrency/pipeline" +require "extensions/dir" -module Zap::Backend - alias Pipeline = ::Zap::Utils::Concurrent::Pipeline +module Backend + alias Pipeline = Concurrency::Pipeline enum Backends CloneFile @@ -12,29 +13,6 @@ module Zap::Backend Symlink end - def self.install(*, dependency : Package, target : Path | String, backend : Backends, store : Store, &on_installing) : Bool - case backend - in .clone_file? - {% if flag?(:darwin) %} - Backend::CloneFile.install(dependency, target, store: store, &on_installing) - {% else %} - raise "The clonefile backend is not supported on this platform" - {% end %} - in .copy_file? - {% if flag?(:darwin) %} - Backend::CopyFile.install(dependency, target, store: store, &on_installing) - {% else %} - raise "The copyfile backend is not supported on this platform" - {% end %} - in .hardlink? - Backend::Hardlink.install(dependency, target, store: store, &on_installing) - in .copy? - Backend::Copy.install(dependency, target, store: store, &on_installing) - in .symlink? - Backend::Symlink.install(dependency, target, store: store, &on_installing) - end - end - # ----------------- # Iterative version # ----------------- @@ -119,10 +97,64 @@ module Zap::Backend # end # end - protected def self.prepare(dependency : Package, dest_path : Path | String, *, store : Store, mkdir_parent = false) : {Path, Path, Bool} + protected def self.prepare(dependency : Data::Package, dest_path : Path | String, *, store : Store) : {Path, Path, Bool} src_path = store.package_path(dependency) - already_installed = Installer.package_already_installed?(dependency, dest_path) - Utils::Directories.mkdir_p(mkdir_parent ? dest_path.dirname : dest_path) unless already_installed + already_installed = self.package_already_installed?(dependency.key, dest_path) + Utils::Directories.mkdir_p(dest_path.dirname) unless already_installed {src_path, dest_path, already_installed} end + + # Check if a package is already installed on the filesystem + def self.package_already_installed?(package_key : String, path : Path) + if exists = Dir.exists?(path) + metadata_path = path / Shared::Constants::METADATA_FILE_NAME + unless File.readable?(metadata_path) + FileUtils.rm_rf(path) + exists = false + else + key = File.read(metadata_path) + if key != package_key + FileUtils.rm_rf(path) + exists = false + end + end + end + exists + end +end + +require "./clonefile" +require "./copy" +require "./copyfile" +require "./hardlink" +require "./symlink" + +module Backend + def self.link(*, dependency : Data::Package, target : Path | String, backend : Backends, store : Store, &on_installing) : Bool + src_path, dest_path, already_installed = self.prepare(dependency, target, store: store) + return false if already_installed + + yield + + case backend + in .clone_file? + {% if flag?(:darwin) %} + Backend::CloneFile.link(src_path, dest_path) + {% else %} + raise "The clonefile backend is not supported on this platform" + {% end %} + in .copy_file? + {% if flag?(:darwin) %} + Backend::CopyFile.link(src_path, dest_path) + {% else %} + raise "The copyfile backend is not supported on this platform" + {% end %} + in .hardlink? + Backend::Hardlink.link(src_path, dest_path) + in .copy? + Backend::Copy.link(src_path, dest_path) + in .symlink? + Backend::Symlink.link(src_path, dest_path) + end + end end diff --git a/packages/backend/clonefile.cr b/packages/backend/clonefile.cr new file mode 100644 index 0000000..d378d4b --- /dev/null +++ b/packages/backend/clonefile.cr @@ -0,0 +1,11 @@ +require "extensions/libc/clonefile" + +module Backend::CloneFile + def self.link(src_path : String | Path, dest_path : String | Path) : Bool + result = LibC.clonefile(src_path.to_s, dest_path.to_s, 0) + if result == -1 + raise "Error cloning file: #{Errno.value} #{src_path.to_s} -> #{dest_path.to_s}" + end + true + end +end diff --git a/packages/backend/copy.cr b/packages/backend/copy.cr new file mode 100644 index 0000000..670d64d --- /dev/null +++ b/packages/backend/copy.cr @@ -0,0 +1,13 @@ +require "concurrency/pipeline" + +module Backend::Copy + def self.link(src_path : String | Path, dest_path : String | Path) : Bool + # FileUtils.cp_r(src_path, dest_path) + Pipeline.wrap(force_wait: true) do |pipeline| + Backend.recursively(src_path.to_s, dest_path.to_s, pipeline: pipeline) do |src, dest| + File.copy(src, dest) + end + end + true + end +end diff --git a/packages/backend/copyfile.cr b/packages/backend/copyfile.cr new file mode 100644 index 0000000..e3255de --- /dev/null +++ b/packages/backend/copyfile.cr @@ -0,0 +1,13 @@ +require "extensions/libc/copyfile" +require "concurrency/pipeline" + +module Backend::CopyFile + def self.link(src_path : String | Path, dest_path : String | Path) : Bool + Pipeline.wrap(force_wait: true) do |pipeline| + Backend.recursively(src_path.to_s, dest_path.to_s, pipeline) do |src, dest| + LibC.copyfile(src.to_s, dest.to_s, nil, LibC::COPYFILE_CLONE_FORCE | LibC::COPYFILE_ALL) + end + end + true + end +end diff --git a/packages/backend/hardlink.cr b/packages/backend/hardlink.cr new file mode 100644 index 0000000..0205edc --- /dev/null +++ b/packages/backend/hardlink.cr @@ -0,0 +1,12 @@ +require "concurrency/pipeline" + +module Backend::Hardlink + def self.link(src_path : String | Path, dest_path : String | Path) : Bool + Pipeline.wrap(force_wait: true) do |pipeline| + Backend.recursively(src_path.to_s, dest_path.to_s, pipeline: pipeline) do |src, dest| + File.link(src, dest) + end + end + true + end +end diff --git a/packages/backend/shard.lock b/packages/backend/shard.lock new file mode 100644 index 0000000..c1b2576 --- /dev/null +++ b/packages/backend/shard.lock @@ -0,0 +1,38 @@ +version: 2.0 +shards: + concurrency: + path: ../concurrency + version: 0.1.0 + + crystar: + git: https://github.com/naqvis/crystar.git + version: 0.4.0 + + data: + path: ../data + version: 0.1.0 + + extensions: + path: ../extensions + version: 0.1.0 + + git: + path: ../git + version: 0.1.0 + + msgpack: + git: https://github.com/crystal-community/msgpack-crystal.git + version: 1.3.4 + + semver: + path: ../semver + version: 0.1.0 + + shared: + path: ../shared + version: 0.1.0 + + utils: + path: ../utils + version: 0.1.0 + diff --git a/packages/backend/shard.yml b/packages/backend/shard.yml new file mode 100644 index 0000000..a6776e9 --- /dev/null +++ b/packages/backend/shard.yml @@ -0,0 +1,12 @@ +name: backend +version: 0.1.0 + +dependencies: + concurrency: + path: ../concurrency + data: + path: ../data + shared: + path: ../shared + extensions: + path: ../extensions diff --git a/packages/backend/symlink.cr b/packages/backend/symlink.cr new file mode 100644 index 0000000..6b85026 --- /dev/null +++ b/packages/backend/symlink.cr @@ -0,0 +1,12 @@ +require "concurrency/pipeline" + +module Backend::Symlink + def self.link(src_path : String | Path, dest_path : String | Path) : Bool + Pipeline.wrap do |pipeline| + Backend.recursively(src_path.to_s, dest_path.to_s, pipeline: pipeline) do |src, dest| + File.symlink(src, dest) + end + end + true + end +end diff --git a/packages/cli/cli.cr b/packages/cli/cli.cr new file mode 100644 index 0000000..6094967 --- /dev/null +++ b/packages/cli/cli.cr @@ -0,0 +1,44 @@ +require "option_parser" +require "core/command_config" +require "core/config" +require "commands/cli" +require "commands/helpers" + +class CLI + @command_config_ref = Core::CommandConfigRef.new + + def initialize( + @commands : Array(Commands::CLI), + @config : Core::Config = Core::Config.new(ENV, "ZAP") + ) + end + + def parse + # Parse options and extract configs + parser = OptionParser.new do |parser| + banner_desc = <<-DESCRIPTION + #{"A package manager for the Javascript language.".colorize.bold} + + Check out #{"https://github.com/elbywan/zap".colorize.magenta} for more information. + DESCRIPTION + Commands::Helpers.banner(parser, "[command]", banner_desc) + + Commands::Helpers.separator("Commands") + @commands.each &.register(parser, @command_config_ref) + + Commands::Helpers.separator("Options") + Commands::Helpers.common_options(true) + Commands::Helpers.workspace_options(true) + end + + parser.parse + + if @command_config_ref.ref.nil? + puts parser + exit + end + + # Return both configs + {@config, @command_config_ref.ref} + end +end diff --git a/packages/cli/shard.lock b/packages/cli/shard.lock new file mode 100644 index 0000000..b5b1315 --- /dev/null +++ b/packages/cli/shard.lock @@ -0,0 +1,70 @@ +version: 2.0 +shards: + backend: + path: ../backend + version: 0.1.0 + + commands: + path: ../commands + version: 0.1.0 + + concurrency: + path: ../concurrency + version: 0.1.0 + + core: + path: ../core + version: 0.1.0 + + crystar: + git: https://github.com/naqvis/crystar.git + version: 0.4.0 + + data: + path: ../data + version: 0.1.0 + + extensions: + path: ../extensions + version: 0.1.0 + + fetch: + path: ../fetch + version: 0.1.0 + + git: + path: ../git + version: 0.1.0 + + msgpack: + git: https://github.com/crystal-community/msgpack-crystal.git + version: 1.3.4 + + reporter: + path: ../reporter + version: 0.1.0 + + semver: + path: ../semver + version: 0.1.0 + + shared: + path: ../shared + version: 0.1.0 + + store: + path: ../store + version: 0.0.1 + + term-cursor: + git: https://github.com/crystal-term/cursor.git + version: 0.1.0+git.commit.8805d5f686d153db92cf2ce3333433f8ed3708d0 + + utils: + path: ../utils + version: 0.1.0 + + workspaces: + path: ../workspaces + version: 0.1.0 + diff --git a/packages/cli/shard.yml b/packages/cli/shard.yml new file mode 100644 index 0000000..252071c --- /dev/null +++ b/packages/cli/shard.yml @@ -0,0 +1,14 @@ +name: cli +version: 0.1.0 +crystalline: + main: zap.cr +targets: + zap: + main: zap.cr +dependencies: + core: + path: ../core + utils: + path: ../utils + commands: + path: ../commands diff --git a/packages/cli/zap.cr b/packages/cli/zap.cr new file mode 100644 index 0000000..c7c6622 --- /dev/null +++ b/packages/cli/zap.cr @@ -0,0 +1,92 @@ +require "./cli" +require "colorize" +require "log" +require "utils/debug_formatter" +require "commands/install/cli" +require "commands/install" +require "commands/dlx/cli" +require "commands/dlx" +require "commands/exec/cli" +require "commands/exec" +require "commands/init/cli" +require "commands/init" +require "commands/rebuild/cli" +require "commands/rebuild" +require "commands/run/cli" +require "commands/run" +require "commands/store/cli" +require "commands/store" +require "commands/why/cli" +require "commands/why" + +module Zap + Colorize.on_tty_only! + Zap.run + + VERSION = {{ `shards version`.stringify }}.chomp + + Log = ::Log.for("zap.entry") + + def self.print_banner + puts "⚡ #{"Zap".colorize.bold.underline} #{"(v#{VERSION})".colorize.dim}" + end + + def self.run + if env = ENV["DEBUG"]? + backend = ::Log::IOBackend.new(STDOUT, formatter: Utils::DebugFormatter) + begin + ::Log.setup(env, level: :debug, backend: backend) + rescue + ::Log.setup_from_env(default_sources: "zap.*", backend: backend) + end + else + ::Log.setup_from_env(default_sources: "zap.*") + end + + begin + Log.debug { "• Registring CLI commands" } + commands = [ + Commands::Install::CLI.new, + Commands::Dlx::CLI.new, + Commands::Exec::CLI.new, + Commands::Init::CLI.new, + Commands::Rebuild::CLI.new, + Commands::Run::CLI.new, + Commands::Store::CLI.new, + Commands::Why::CLI.new, + ].map(&.as(Commands::CLI)) + Log.debug { "• Parsing the CLI arguments" } + config, command_config = CLI.new(commands).parse + rescue e + puts e.message + exit Shared::Constants::ErrorCodes::EARLY_EXIT.to_i32 + end + + Log.debug { "• Executing command #{command_config.to_s}" } + + case command_config + when Commands::Install::Config + Commands::Install.run(config, command_config) + when Commands::Dlx::Config + Commands::Dlx.run(config, command_config) + when Commands::Init::Config + Commands::Init.run(config, command_config) + when Commands::Run::Config + script_name = ARGV[0]? + args = ARGV[1..-1]? || Array(String).new + command_config = command_config.copy_with(script: script_name, args: args) + Commands::Run.run(config, command_config) + when Commands::Rebuild::Config + Commands::Rebuild.run(config, command_config) + when Commands::Exec::Config + command_config = command_config.copy_with(command: ARGV[0] || "", args: ARGV[1..-1]) + Commands::Exec.run(config, command_config) + when Commands::Store::Config + Commands::Store.run(config, command_config) + when Commands::Why::Config + Commands::Why.run(config, command_config) + else + raise "Unknown command config: #{command_config}" + end + end +end diff --git a/packages/commands/cli.cr b/packages/commands/cli.cr new file mode 100644 index 0000000..af2ec80 --- /dev/null +++ b/packages/commands/cli.cr @@ -0,0 +1,5 @@ +require "option_parser" + +abstract class Commands::CLI + abstract def register(parser : OptionParser, config : Core::CommandConfigRef) : Nil +end diff --git a/packages/commands/dlx/cli.cr b/packages/commands/dlx/cli.cr new file mode 100644 index 0000000..ec02b81 --- /dev/null +++ b/packages/commands/dlx/cli.cr @@ -0,0 +1,33 @@ +require "../cli" +require "../helpers" +require "./config" + +class Commands::Dlx::CLI < Commands::CLI + def register(parser : OptionParser, command_config : Core::CommandConfigRef) : Nil + Helpers.command(["dlx", "x"], "Install one or more packages and run a command in a temporary environment.", "[options] ") do + command_config.ref = Config.new(ENV, "ZAP_DLX") + + Helpers.separator("Options") + + Helpers.flag("-c ", "--call ", "Runs the command inside of a shell.") do |command| + command_config.ref = dlx_config.copy_with(call: command) + end + Helpers.flag("-p ", "--package ", "The package or packages to install.") do |package| + dlx_config.packages << package + end + Helpers.flag("-q", "--quiet", "Mute most of the output coming from zap. #{"[env: ZAP_DLX_QUIET]".colorize.dim}") do |package| + command_config.ref = dlx_config.copy_with(quiet: true) + end + + parser.before_each do |arg| + unless arg.starts_with?("-") + parser.stop + end + end + end + end + + private macro dlx_config + command_config.ref.as(Dlx::Config) + end +end diff --git a/src/commands/dlx/config.cr b/packages/commands/dlx/config.cr similarity index 88% rename from src/commands/dlx/config.cr rename to packages/commands/dlx/config.cr index 9f0ca7c..69f5e15 100644 --- a/src/commands/dlx/config.cr +++ b/packages/commands/dlx/config.cr @@ -1,7 +1,7 @@ -require "../config" -require "../../utils/macros" +require "utils/macros" +require "core/command_config" -struct Zap::Commands::Dlx::Config < Zap::Commands::Config +struct Commands::Dlx::Config < Core::CommandConfig Utils::Macros.record_utils SPACE_REGEX = /\s+/ diff --git a/src/commands/dlx/dlx.cr b/packages/commands/dlx/dlx.cr similarity index 78% rename from src/commands/dlx/dlx.cr rename to packages/commands/dlx/dlx.cr index 2bb49a5..0ad8dce 100644 --- a/src/commands/dlx/dlx.cr +++ b/packages/commands/dlx/dlx.cr @@ -1,12 +1,16 @@ +require "log" +require "shared/constants" require "./config" -module Zap::Commands::Dlx +module Commands::Dlx + Log = ::Log.for("zap.commands.dlx") + def self.run( - config : Zap::Config, + config : Core::Config, dlx_config : Dlx::Config ) dlx_config = dlx_config.from_args(ARGV) - main_package = Package.init?(Path.new(config.prefix)) + main_package = Data::Package.init?(Path.new(config.prefix)) packages = get_packages_versions(main_package, dlx_config.packages) Log.debug { "Packages versions: #{packages}" } @@ -22,13 +26,13 @@ module Zap::Commands::Dlx ) # If the folder is already sealed, we can skip the installation part - unless File.exists?(path / Installer::METADATA_FILE_NAME) - Log.debug { "Location #{path / Installer::METADATA_FILE_NAME} does not exist." } + unless File.exists?(path / Shared::Constants::METADATA_FILE_NAME) + Log.debug { "Location #{path / Shared::Constants::METADATA_FILE_NAME} does not exist." } Log.debug { "Installing packages…" } FileUtils.rm_rf(path) Dir.mkdir(path) # Make a fictional package.json - pkg_json = Package.new(directory_name, "0.0.0") + pkg_json = Data::Package.new(directory_name, "0.0.0") # Add to the package the requested packages pkg_json.dependencies = packages.to_h # Write the package.json @@ -43,13 +47,13 @@ module Zap::Commands::Dlx puts "" # Seal the folder - File.touch(path / Installer::METADATA_FILE_NAME) + File.touch(path / Shared::Constants::METADATA_FILE_NAME) else - Log.debug { "Location #{path / Installer::METADATA_FILE_NAME} exists. Skipping installation phase." } + Log.debug { "Location #{path / Shared::Constants::METADATA_FILE_NAME} exists. Skipping installation phase." } end if dlx_config.command.empty? - pkg_json = Package.init(path / "node_modules" / packages[0][0]) + pkg_json = Data::Package.init(path / "node_modules" / packages[0][0]) if bin = pkg_json.bin unscoped_name = pkg_json.name.split('/').last if bin.is_a?(String) @@ -82,11 +86,11 @@ module Zap::Commands::Dlx # Get the packages and versions from the config def self.get_packages_versions( - main_package : Package?, + main_package : Data::Package?, config_packages : Array(String) - ) : Array({String, String | Zap::Package::Alias}) + ) : Array({String, String | Data::Package::Alias}) config_packages.map do |package| - name, version = Utils::Various.parse_key(package) + name, version = Utils::Misc.parse_key(package) # If the version is not specified, check if the package json has the package and get the version. Otherwise assume *. version ||= main_package.try do |pkg| pkg.dependencies.try &.[name]? || @@ -99,7 +103,7 @@ module Zap::Commands::Dlx end # Infer an identifier and a suitable location from the packages and versions - def self.get_identifier_and_path(packages : Array({String, String | Zap::Package::Alias}), *, prefix : String? = nil) + def self.get_identifier_and_path(packages : Array({String, String | Data::Package::Alias}), *, prefix : String? = nil) identifier = Digest::SHA1.hexdigest(packages.map { |p| "#{p[0]}@#{p[1]}" }.join("+")) directory = "#{prefix}#{identifier}" path = Path.new(Dir.tempdir) / directory diff --git a/packages/commands/exec/cli.cr b/packages/commands/exec/cli.cr new file mode 100644 index 0000000..0d00b96 --- /dev/null +++ b/packages/commands/exec/cli.cr @@ -0,0 +1,27 @@ +require "../cli" +require "../helpers" +require "./config" + +class Commands::Exec::CLI < Commands::CLI + def register(parser : OptionParser, command_config : Core::CommandConfigRef) : Nil + Helpers.command(["exec", "e"], "Execute a command in the project scope.", "") do + command_config.ref = Exec::Config.new(ENV, "ZAP_EXEC") + + Helpers.separator("Options") + + Helpers.flag("--parallel", "Run all commands in parallel without any kind of topological ordering. #{"[env: ZAP_EXEC_PARALLEL]".colorize.dim}") do + command_config.ref = exec_config.copy_with(parallel: true) + end + + parser.before_each do |arg| + unless arg.starts_with?("-") + parser.stop + end + end + end + end + + private macro exec_config + command_config.ref.as(Exec::Config) + end +end diff --git a/src/commands/exec/config.cr b/packages/commands/exec/config.cr similarity index 58% rename from src/commands/exec/config.cr rename to packages/commands/exec/config.cr index 002f6ef..8161496 100644 --- a/src/commands/exec/config.cr +++ b/packages/commands/exec/config.cr @@ -1,7 +1,7 @@ -require "../config" -require "../../utils/macros" +require "utils/macros" +require "core/command_config" -struct Zap::Commands::Exec::Config < Zap::Commands::Config +struct Commands::Exec::Config < Core::CommandConfig Utils::Macros.record_utils getter command : String = "" diff --git a/src/commands/exec/exec.cr b/packages/commands/exec/exec.cr similarity index 51% rename from src/commands/exec/exec.cr rename to packages/commands/exec/exec.cr index ce6658d..19b57f5 100644 --- a/src/commands/exec/exec.cr +++ b/packages/commands/exec/exec.cr @@ -1,5 +1,11 @@ -module Zap::Commands::Exec - def self.run(config : Zap::Config, exec_config : Exec::Config, *, no_banner : Bool = false) +require "core/config" +require "reporter/interactive" +require "./config" + +module Commands::Exec + Log = ::Log.for("zap.exec") + + def self.run(config : Core::Config, exec_config : Exec::Config, *, no_banner : Bool = false) reporter = Reporter::Interactive.new begin raise "Please provide a command to run. Example: 'zap exec '" if exec_config.command.empty? @@ -9,6 +15,8 @@ module Zap::Commands::Exec workspaces, config = inferred_context.workspaces, inferred_context.config targets = inferred_context.scope_packages_and_paths(:command) + return if targets.size == 0 + unless config.silent || no_banner Zap.print_banner if workspaces @@ -16,11 +24,13 @@ module Zap::Commands::Exec #{"scope".colorize.blue}: #{inferred_context.command_scope.size} package(s) • #{targets.map(&.[0].name).sort.join(", ")} TERM end - print NEW_LINE + print Shared::Constants::NEW_LINE end + Log.debug { "• Initializing scripts" } + scripts = targets.map do |package, path| - Utils::Scripts::ScriptData.new( + Data::Package::Scripts::ScriptData.new( package, path, "[exec]", @@ -29,23 +39,33 @@ module Zap::Commands::Exec ) end - workspace_relationships = workspaces.try(&.relationships) + Log.debug { "• Running the scripts" } - if !workspace_relationships || exec_config.parallel - Utils::Scripts.parallel_run( + if targets.size < 2 || exec_config.parallel + Data::Package::Scripts.parallel_run( config: config, scripts: scripts, reporter: reporter, print_header: false, ) else - Utils::Scripts.topological_run( - config: config, - scripts: scripts, - relationships: workspace_relationships, - reporter: reporter, - print_header: false, - ) + workspace_relationships = workspaces.try(&.relationships) + if !workspace_relationships + Data::Package::Scripts.parallel_run( + config: config, + scripts: scripts, + reporter: reporter, + print_header: false, + ) + else + Data::Package::Scripts.topological_run( + config: config, + scripts: scripts, + relationships: workspace_relationships, + reporter: reporter, + print_header: false, + ) + end end rescue ex : Exception reporter.error(ex) diff --git a/packages/commands/helpers.cr b/packages/commands/helpers.cr new file mode 100644 index 0000000..27ae981 --- /dev/null +++ b/packages/commands/helpers.cr @@ -0,0 +1,173 @@ +require "extensions/option_parser" + +module Commands::Helpers + def self.banner(parser, command, description, *, args = "[options]") + parser.banner = <<-BANNER + ⚡ #{"Zap".colorize.yellow.bold.underline} #{"(v#{Zap::VERSION})".colorize.dim} + + #{description} + + #{"Usage".colorize.underline.magenta.bold}: zap #{command} #{args} + #{" zap [command] --help for more information on a specific command".colorize.dim} + BANNER + end + + macro common_options(sub = false) + {% if sub %}Commands::Helpers.subSeparator{% else %}Commands::Helpers.separator{% end %}("Common", early_line_break: false) + + Commands::Helpers.flag("-C ", "--dir ", "Use PATH as the root directory of the project. #{"[env: ZAP_PREFIX]".colorize.dim}") do |path| + @config = @config.copy_with(prefix: Path.new(path).expand.to_s, global: false) + end + + Commands::Helpers.flag("--concurrency ", "Set the maximum number of tasks that will be run in parallel. (default: 5) #{"[env: ZAP_CONCURRENCY]".colorize.dim}") do |concurrency| + @config = @config.copy_with(concurrency: concurrency.to_i32) + end + + Commands::Helpers.flag("--deferred-output", "Do not print the output in real time when running multiple scripts in parallel but instead defer it to have a nicer packed output. (default: false unless CI) #{"[env: ZAP_DEFERRED_OUTPUT]".colorize.dim}") do + @config = @config.copy_with(deferred_output: true) + end + + Commands::Helpers.flag( + "--file-backend ", + <<-DESCRIPTION + The backend to use when linking packages on disk. #{"[env: ZAP_FILE_BACKEND]".colorize.dim} + Possible values: + - clonefile (default on macOS - macOS only) + - hardlink (default on linux) + - copyfile (macOS only) + - copy (default fallback) + DESCRIPTION + ) do |backend| + @config = @config.copy_with(file_backend: Backend::Backends.parse(backend)) + end + + Commands::Helpers.flag( + "--flock-scope ", + <<-DESCRIPTION + Set the scope of the file lock mechanism used to prevent store corruption. #{"[env: ZAP_FLOCK_SCOPE]".colorize.dim} + Possible values: + - global (default) : The lock is global to the whole store. Slower, but will not hit the maximum number of open files limit. + - package : The lock is scoped to the current package. Faster, but may hit the default maximum number of open files limit. + - none : No flock lock is used. Faster, but will not work if multiple Zap processes are running in parallel. + DESCRIPTION + ) do |scope| + @config = @config.copy_with(flock_scope: Core::Config::FLockScope.parse(scope)) + end + + Commands::Helpers.flag("-g", "--global", "Operates in \"global\" mode, so that packages are installed into the global folder instead of the current working directory.") do |path| + @config = @config.copy_with(prefix: @config.deduce_global_prefix, global: true) + end + + Commands::Helpers.flag("-h", "--help", "Show this help.") do + puts parser + exit + end + + Commands::Helpers.flag("--silent", "Minimize the output. #{"[env: ZAP_SILENT]".colorize.dim}") do + @config = @config.copy_with(silent: true) + end + + Commands::Helpers.flag("-v", "--version", "Show version.") do + puts "v#{Zap::VERSION}" + exit + end + + Commands::Helpers.flag("--lockfile-format ", "The serialization to use when saving the lockfile to the disk. (Default: the current lockfile format, or YAML) #{"[env: ZAP_LOCKFILE_FORMAT]".colorize.dim}") do |format| + @config = @config.copy_with(lockfile_format: Data::Lockfile::Format.parse(format)) + end + end + + macro workspace_options(sub = false) + {% if sub %}Commands::Helpers.subSeparator{% else %}Commands::Helpers.separator{% end %}("Workspace") + + Commands::Helpers.flag("-F ", "--filter ", "Filtering allows you to restrict commands to specific subsets of packages.") do |filter| + filters = @config.filters || Array(Workspaces::Filter).new + filters << Workspaces::Filter.new(filter) + @config = @config.copy_with(filters: filters) + end + + Commands::Helpers.flag("--ignore-workspaces", "Will completely ignore workspaces when applying the command. #{"[env: ZAP_NO_WORKSPACES]".colorize.dim}") do + @config = @config.copy_with(no_workspaces: true) + end + + Commands::Helpers.flag("-r", "--recursive", "Will apply the command to all packages in the workspace. #{"[env: ZAP_RECURSIVE]".colorize.dim}") do + @config = @config.copy_with(recursive: true) + end + + Commands::Helpers.flag("-w", "--workspace-root", "Will apply the command to the root workspace package. #{"[env: ZAP_ROOT_WORKSPACE]".colorize.dim}") do + @config = @config.copy_with(root_workspace: true) + end + end + + macro separator(text, *, prepend = false) + %text = "\n#{ {{text + ":"}}.colorize.underline }\n".colorize.green.bold.to_s + {% if prepend %} + parser.@flags.unshift(%text) + {% else %} + parser.separator(%text) + {% end %} + end + + macro subSeparator(text, *, early_line_break = true) + prefix = "#{ {% if early_line_break %}Shared::Constants::NEW_LINE{% else %}nil{% end %} }" + parser.separator("#{prefix} #{ {{text}} }\n".colorize.blue.bold) + end + + # @command_color_index = 0 + + def self.command_formatter(flag) + # flag.colorize(COLORS[@command_color_index % COLORS.size]).bold.tap { + # @command_color_index += 1 + # }.to_s + flag.colorize.bold.to_s + end + + def self.flag_formatter(flag) + flag + end + + macro flag(*args, &block) + parser.on(*{{ args }}, ->Commands::Helpers.flag_formatter(String)){{ block }} + end + + macro command(input, description, args = nil) + {% if input.is_a?(StringLiteral) %} + parser.on({{input}},{{description}}, ->::Commands::Helpers.command_formatter(String)) do + ::Commands::Helpers.banner(parser, {{input}}, {{description}} {% if args %}, args: {{args}}{% end %}) + ::Commands::Helpers.separator("Inherited", prepend: true) + %flags_bak = parser.@flags.dup + parser.@flags.clear + {{ yield }} + %flags_bak.each { |flag| parser.@flags << flag } + end + {% else %} + {% for a, idx in input %} + %aliases = {{input}}[...{{idx}}] + {{input}}[({{idx}} + 1)...] + %desc = {{description}} + %( alias(es): #{%aliases.join(", ")}).colorize.dim.to_s + {% if idx == 0 %} + parser.on({{a}}, %desc, ->::Commands::Helpers.command_formatter(String)) do + %aliases = {{input}}[...{{idx}}] + {{input}}[({{idx}} + 1)...] + %desc = {{description}} + %(\nalias(es): #{%aliases.join(", ")}).colorize.dim.to_s + ::Commands::Helpers.banner(parser, {{a}}, %desc {% if args %}, args: {{args}}{% end %}) + ::Commands::Helpers.separator("Inherited", prepend: true) + %flags_bak = parser.@flags.dup + parser.@flags.clear + {{ yield }} + %flags_bak.each { |flag| parser.@flags << flag } + end + {% else %} + parser.on({{a}}, %desc, no_help_text: true) do + %aliases = {{input}}[...{{idx}}] + {{input}}[({{idx}} + 1)...] + %desc = {{description}} + %(\nalias(es): #{%aliases.join(", ")}).colorize.dim.to_s + ::Commands::Helpers.banner(parser, {{a}}, %desc {% if args %}, args: {{args}}{% end %}) + ::Commands::Helpers.separator("Inherited", prepend: true) + %flags_bak = parser.@flags.dup + parser.@flags.clear + {{ yield }} + %flags_bak.each { |flag| parser.@flags << flag } + end + {% end %} + {% end %} + {% end %} + end +end diff --git a/packages/commands/init/cli.cr b/packages/commands/init/cli.cr new file mode 100644 index 0000000..a531c29 --- /dev/null +++ b/packages/commands/init/cli.cr @@ -0,0 +1,45 @@ +require "../cli" +require "../helpers" +require "../dlx/config" +require "./config" + +class Commands::Init::CLI < Commands::CLI + def register(parser : OptionParser, command_config : Core::CommandConfigRef) : Nil + Helpers.command(["init", "innit", "create"], "Create a new package.json file.", "[options] ") do + command_config.ref = Init::Config.new(ENV, "ZAP_INIT") + + Helpers.separator("Options") + + Helpers.flag("-y", "--yes", %(Automatically answer "yes" to any prompts that zap might print on the command line. #{"[env: ZAP_INIT_YES]".colorize.dim})) do |package| + command_config.ref = init_config.copy_with(yes: true) + end + + parser.before_each do |arg| + unless arg.starts_with?("-") + if (arg.starts_with?("@")) + split_arg = arg[1..].split('@') + slash_split = split_arg.first.split('/') + package_descriptor = "@#{slash_split.join("/create-")}" + command = "create-#{slash_split[1]}" + version = split_arg[1]? ? "@#{split_arg[1]}" : "" + @command_config = Dlx::Config.new( + packages: [package_descriptor], + create_command: command + ) + else + package_descriptor = "create-#{arg}" + @command_config = Dlx::Config.new( + packages: [package_descriptor], + create_command: package_descriptor.split('@').first + ) + end + parser.stop + end + end + end + end + + private macro init_config + command_config.ref.as(Init::Config) + end +end diff --git a/packages/commands/init/config.cr b/packages/commands/init/config.cr new file mode 100644 index 0000000..cfd6f70 --- /dev/null +++ b/packages/commands/init/config.cr @@ -0,0 +1,9 @@ +require "utils/macros" +require "core/command_config" + +struct Commands::Init::Config < Core::CommandConfig + Utils::Macros.record_utils + + @[Env] + getter yes : Bool = !STDIN.tty? +end diff --git a/src/commands/init/init.cr b/packages/commands/init/init.cr similarity index 98% rename from src/commands/init/init.cr rename to packages/commands/init/init.cr index 8a10008..f709b32 100644 --- a/src/commands/init/init.cr +++ b/packages/commands/init/init.cr @@ -1,4 +1,4 @@ -module Zap::Commands::Init +module Commands::Init macro prompt(description, field_name) unless init_config.yes print "#{ {{ description }} }: (#{ {{ field_name }} }) " @@ -12,7 +12,7 @@ module Zap::Commands::Init end def self.run( - config : Zap::Config, + config : Core::Config, init_config : Init::Config ) prefix_path = Path.new(config.prefix) diff --git a/packages/commands/install/cli.cr b/packages/commands/install/cli.cr new file mode 100644 index 0000000..64dfe4f --- /dev/null +++ b/packages/commands/install/cli.cr @@ -0,0 +1,124 @@ +require "core/command_config" +require "utils/misc" +require "shared/constants" +require "../cli" +require "../helpers" +require "./config" + +class Commands::Install::CLI < Commands::CLI + def register(parser : OptionParser, command_config : Core::CommandConfigRef) : Nil + Helpers.command(["install", "i", "add"], "Install one or more packages and any packages that they depends on.", "[options] ") do + on_install(parser, command_config) + end + + Helpers.command(["remove", "rm", "uninstall", "un"], "Remove one or more packages from the node_modules folder, the package.json file and the lockfile.", "[options] ") do + on_install(parser, command_config, remove_packages: true) + end + + Helpers.command(["update", "up", "upgrade"], "Update the lockfile to use the newest package versions.", "[options] ") do + on_install(parser, command_config, update_packages: true) + end + end + + def on_install( + parser : OptionParser, + command_config : Core::CommandConfigRef, + *, + update_packages : Bool = false, + remove_packages : Bool = false, + ) : Nil + command_config.ref = Config.new(ENV, "ZAP_INSTALL").copy_with(update_all: update_packages) + + Helpers.separator("Options") + + Helpers.flag("--frozen-lockfile ", "If true, will fail if the lockfile is outdated. #{"[env: ZAP_INSTALL_FROZEN_LOCKFILE]".colorize.dim}") do |frozen_lockfile| + command_config.ref = install_config.copy_with(frozen_lockfile: Utils::Misc.str_to_bool(frozen_lockfile)) + end + Helpers.flag("--ignore-scripts", "If true, does not run scripts specified in package.json files. #{"[env: ZAP_INSTALL_IGNORE_SCRIPTS]".colorize.dim}") do + command_config.ref = install_config.copy_with(ignore_scripts: true) + end + Helpers.flag("--no-logs", "If true, will not print logs like deprecation warnings. #{"[env: ZAP_INSTALL_PRINT_LOGS=false]".colorize.dim}") do + command_config.ref = install_config.copy_with(print_logs: false) + end + Helpers.flag("--peers", "Pass this flag to enable checking for missing peer dependencies. #{"[env: ZAP_INSTALL_CHECK_PEER_DEPENDENCIES]".colorize.dim}") do + command_config.ref = install_config.copy_with(check_peer_dependencies: true) + end + Helpers.flag("--prefer-offline", "Bypass staleness checks for package metadata cached from the registry. #{"[env: ZAP_INSTALL_PREFER_OFFLINE]".colorize.dim}") do + command_config.ref = install_config.copy_with(prefer_offline: true) + end + Helpers.flag("--production", "If true, will not install devDependencies.") do + command_config.ref = install_config.copy_with(omit: [Commands::Install::Config::Omit::Dev]) + end + + Helpers.subSeparator("Strategies") + + Helpers.flag( + "--install-strategy ", + <<-DESCRIPTION + The strategy used to install packages. #{"[env: ZAP_INSTALL_STRATEGY]".colorize.dim} + Possible values: + - classic (default) : mimics the behavior of npm and yarn: install non-duplicated in top-level, and duplicated as necessary within directory structure. + - isolated : mimics the behavior of pnpm: dependencies are symlinked from a virtual store at node_modules/.zap. + - pnp : a limited plug'n'play approach similar to yarn + - classic_shallow : like classic but will only install direct dependencies at top-level. + DESCRIPTION + ) do |strategy| + command_config.ref = install_config.copy_with(strategy: Data::Package::InstallStrategy.parse(strategy)) + end + + Helpers.flag("--classic", "Shorthand for: --install-strategy classic") do + command_config.ref = install_config.copy_with(strategy: Data::Package::InstallStrategy::Classic) + end + + Helpers.flag("--isolated", "Shorthand for: --install-strategy isolated") do + command_config.ref = install_config.copy_with(strategy: Data::Package::InstallStrategy::Isolated) + end + + Helpers.flag("--pnp", "Shorthand for: --install-strategy pnp") do + command_config.ref = install_config.copy_with(strategy: Data::Package::InstallStrategy::Pnp) + end + + Helpers.subSeparator("Save") + + unless update_packages + Helpers.flag("--no-save", "Prevents saving to dependencies. #{"[env: ZAP_INSTALL_SAVE=false]".colorize.dim}") do + command_config.ref = install_config.copy_with(save: false) + end + Helpers.flag("-D", "--save-dev", "Added packages will appear in your devDependencies. #{"[env: ZAP_INSTALL_SAVE_DEV]".colorize.dim}") do + command_config.ref = install_config.copy_with(save_dev: true) + end + Helpers.flag("-E", "--save-exact", "Saved dependencies will be configured with an exact version rather than using npm's default semver range operator. #{"[env: ZAP_INSTALL_SAVE_EXACT]".colorize.dim}") do |path| + command_config.ref = install_config.copy_with(save_exact: true) + end + Helpers.flag("-O", "--save-optional", "Added packages will appear in your optionalDependencies. #{"[env: ZAP_INSTALL_SAVE_OPTIONAL]".colorize.dim}") do + command_config.ref = install_config.copy_with(save_optional: true) + end + Helpers.flag("-P", "--save-prod", "Added packages will appear in your dependencies. #{"[env: ZAP_INSTALL_SAVE_PROD]".colorize.dim}") do + command_config.ref = install_config.copy_with(save_prod: true) + end + end + + parser.missing_option do |option| + if option == "--frozen-lockfile" + command_config.ref = install_config.copy_with(frozen_lockfile: true) + else + raise OptionParser::MissingOption.new(option) + end + end + + parser.unknown_args do |pkgs| + if remove_packages + install_config.removed_packages.concat(pkgs) + elsif update_packages + command_config.ref = install_config.copy_with(update_all: pkgs.size == 0) + install_config.updated_packages.concat(pkgs) + else + install_config.added_packages.concat(pkgs) + end + end + end + + private macro install_config + command_config.ref.as(Install::Config) + end +end diff --git a/src/commands/install/config.cr b/packages/commands/install/config.cr similarity index 78% rename from src/commands/install/config.cr rename to packages/commands/install/config.cr index c6e633d..b1766b7 100644 --- a/src/commands/install/config.cr +++ b/packages/commands/install/config.cr @@ -1,7 +1,8 @@ -require "../config" -require "../../utils/macros" +require "core/command_config" +require "utils/macros" +require "data/package" -struct Zap::Commands::Install::Config < Zap::Commands::Config +struct Commands::Install::Config < Core::CommandConfig Utils::Macros.record_utils enum Omit @@ -10,20 +11,13 @@ struct Zap::Commands::Install::Config < Zap::Commands::Config Peer end - enum InstallStrategy - Classic - Classic_Shallow - Isolated - Pnp - end - # Configuration specific for the install command @[Env] getter frozen_lockfile : Bool = !!ENV["CI"]? @[Env] getter ignore_scripts : Bool = false @[Env] - getter! strategy : InstallStrategy + getter! strategy : Data::Package::InstallStrategy getter omit : Array(Omit) = ENV["NODE_ENV"]? === "production" ? [Omit::Dev] : [] of Omit getter added_packages : Array(String) = Array(String).new getter removed_packages : Array(String) = Array(String).new @@ -65,11 +59,11 @@ struct Zap::Commands::Install::Config < Zap::Commands::Config omit.includes?(Omit::Peer) end - def merge_lockfile(lockfile : Lockfile) - self.copy_with(strategy: @strategy || lockfile.strategy || InstallStrategy::Classic) + def merge_lockfile(lockfile : Data::Lockfile) + self.copy_with(strategy: @strategy || lockfile.strategy || Data::Package::InstallStrategy::Classic) end - def merge_pkg(package : Package) + def merge_pkg(package : Data::Package) self.copy_with( strategy: @strategy || package.zap_config.try(&.strategy) || nil, check_peer_dependencies: @check_peer_dependencies || package.zap_config.try(&.check_peer_dependencies) || false, diff --git a/src/commands/install/install.cr b/packages/commands/install/install.cr similarity index 76% rename from src/commands/install/install.cr rename to packages/commands/install/install.cr index cef5c12..974a712 100644 --- a/src/commands/install/install.cr +++ b/packages/commands/install/install.cr @@ -1,29 +1,38 @@ +require "benchmark" +require "log" +require "concurrency/pipeline" +require "reporter/reporter" +require "reporter/null" +require "reporter/interactive" +require "store" +require "extensions/scheduler" +require "data/package/scripts" +require "utils/shasum" require "./config" -require "../../config" -require "../../npmrc" -require "./resolver" require "./state" -require "./registry_clients" -require "../../installer/isolated" -require "../../installer/classic" -require "../../installer/pnp" -require "../../workspaces" +require "./resolver" +require "./linker" +require "./linker/classic" +require "./linker/isolated" +require "./linker/pnp" -module Zap::Commands::Install - alias Pipeline = Utils::Concurrent::Pipeline +module Commands::Install + Log = ::Log.for("zap.commands.install") + + alias Pipeline = Concurrency::Pipeline def self.run( - config : Zap::Config, + config : Core::Config, install_config : Install::Config, *, reporter : Reporter? = nil, - store : Zap::Store? = nil, - raise_on_failure : Bool = false + store : ::Store? = nil, + raise_on_failure : Bool = false, ) state = uninitialized State reporter ||= config.silent ? Reporter::Null.new : Reporter::Interactive.new config = config.check_if_store_is_linkeable - store ||= Zap::Store.new(config.store_path) + store ||= ::Store.new(config.store_path) unmet_peers_hash = nil Zap.print_banner unless config.silent @@ -35,7 +44,7 @@ module Zap::Commands::Install Log.debug { "Configuration: #{config.pretty_inspect}" } - lockfile = Lockfile.new(config.prefix, default_format: config.lockfile_format) + lockfile = Data::Lockfile.new(config.prefix, default_format: config.lockfile_format) # Merge zap config from package.json and lockfile install_config = install_config @@ -45,12 +54,12 @@ module Zap::Commands::Install Log.debug { "Install Configuration: #{install_config.pretty_inspect}" } # Load .npmrc file - npmrc = Npmrc.new(config.prefix) + npmrc = Data::Npmrc.new(config.prefix) Log.debug { "Npmrc: #{npmrc.pretty_inspect}" } # Raise if frozen lockfile is set and the lockfile is not found if install_config.frozen_lockfile && !lockfile.read_status.from_disk? - raise "The --frozen-lockfile flag is on but the lockfile is missing. Run `zap i --frozen-lockfile=false` to generate the lockfile and try again." + raise "The --frozen-lockfile flag is on but the lockfile is missing.\nRun `zap i --frozen-lockfile=false` to generate the lockfile and try again." end # Print info about the install @@ -60,16 +69,16 @@ module Zap::Commands::Install config = self.strategy_check(config, install_config, lockfile, inferred_context, reporter) # Force hoisting if the hoisting options have changed - self.hoisting_check(install_config, lockfile, inferred_context, reporter) + install_config = self.hoisting_check(install_config, lockfile, inferred_context, reporter) # Force metadata retrieval if the package extensions options have changed - self.package_extensions_check(install_config, lockfile, inferred_context, reporter) + install_config = self.package_extensions_check(install_config, lockfile, inferred_context, reporter) # Init state struct state = State.new( config: config, install_config: config.global ? install_config.copy_with( - strategy: Config::InstallStrategy::Classic_Shallow + strategy: Data::Package::InstallStrategy::Classic_Shallow ) : install_config, store: store, main_package: inferred_context.main_package, @@ -106,14 +115,21 @@ module Zap::Commands::Install if state.install_config.frozen_lockfile # Raise if the lockfile has been updated - if (state.lockfile.serialize != File.read(state.lockfile.lockfile_path)) - raise "The --frozen-lockfile flag is on but the lockfile has been updated during the resolution phase. Run `zap i --frozen-lockfile=false` to regenerate the lockfile and try again." + + current_shasum = Shasum.new(Digest::SHA1.new).tap do |shasum| + state.lockfile.serialize(shasum) + end.final + new_shasum = Digest::SHA1.new.file(state.lockfile.lockfile_path).final + + if (current_shasum != new_shasum) + raise "The --frozen-lockfile flag is on but the lockfile has been updated during the resolution phase.\nRun `zap i --frozen-lockfile=false` to regenerate the lockfile and try again." end end # Do not edit lockfile or package.json files in global mode or if the save flag is false unless state.config.global || !state.install_config.save # Write lockfile + Log.debug { "• Writing the lockfile" } state.lockfile.write(format: config.lockfile_format) # Edit and write the package.json files if the flags have been set in the config @@ -121,10 +137,10 @@ module Zap::Commands::Install end # Install dependencies to the appropriate node_modules folder - installer = install_packages(state, pruned_direct_dependencies) + linker = link_packages(state, pruned_direct_dependencies) # Run package.json hooks for the installed packages - run_install_hooks(state, installer) + run_install_hooks(state, linker) # Run package.json hooks for the workspace packages run_own_install_hooks(state) @@ -135,7 +151,7 @@ module Zap::Commands::Install rescue e raise e if raise_on_failure reporter.try &.error(e) - exit ErrorCodes::INSTALL_COMMAND_FAILED.to_i32 + exit Shared::Constants::ErrorCodes::INSTALL_COMMAND_FAILED.to_i32 end # -PRIVATE--------------------------- # @@ -151,11 +167,11 @@ module Zap::Commands::Install end private def self.print_info( - config : Zap::Config, - inferred_context : Zap::Config::InferredContext, + config : Core::Config, + inferred_context : Core::Config::InferredContext, install_config : Install::Config, - lockfile : Lockfile, - workspaces : Workspaces? + lockfile : Data::Lockfile, + workspaces : Workspaces?, ) unless config.silent workers_info = begin @@ -166,16 +182,16 @@ module Zap::Commands::Install {% end %} end puts <<-TERM - #{"project:".colorize.blue} #{config.prefix} • #{"store:".colorize.blue} #{config.store_path}#{workers_info} - #{"lockfile:".colorize.blue} #{lockfile.read_status.from_disk? ? "ok".colorize.green : lockfile.read_status.error? ? "read error".colorize.red : "not found".colorize.red} #{"[#{lockfile.format}]".colorize.italic.dim} • #{"install strategy:".colorize.blue} #{install_config.strategy.to_s.downcase} - TERM + #{"project:".colorize.blue} #{config.prefix} • #{"store:".colorize.blue} #{config.store_path}#{workers_info} + #{"lockfile:".colorize.blue} #{lockfile.read_status.from_disk? ? "ok".colorize.green : lockfile.read_status.error? ? "read error".colorize.red : "not found".colorize.red} #{"[#{lockfile.format}]".colorize.italic.dim} • #{"install strategy:".colorize.blue} #{install_config.strategy.to_s.downcase} + TERM if workspaces install_scope_packages = inferred_context.scope_names(:install).sort.join(", ") suffix = install_scope_packages.size > 0 ? " • #{install_scope_packages}" : "" puts <<-TERM - #{"install scope".colorize.blue}: #{inferred_context.install_scope.size} package(s)#{suffix} - TERM + #{"install scope".colorize.blue}: #{inferred_context.install_scope.size} package(s)#{suffix} + TERM end if ( @@ -185,20 +201,20 @@ module Zap::Commands::Install command_scope_packages = inferred_context.scope_names(:command).sort.join(", ") suffix = command_scope_packages.size > 0 ? " • #{command_scope_packages}" : "" puts <<-TERM - #{"add/remove scope".colorize.blue}: #{inferred_context.command_scope.size} package(s)#{suffix} - TERM + #{"add/remove scope".colorize.blue}: #{inferred_context.command_scope.size} package(s)#{suffix} + TERM end puts end end private def self.strategy_check( - config : Zap::Config, + config : Core::Config, install_config : Install::Config, - lockfile : Lockfile, - context : Zap::Config::InferredContext, - reporter : Reporter - ) : Zap::Config + lockfile : Data::Lockfile, + context : Core::Config::InferredContext, + reporter : Reporter, + ) : Core::Config if !config.global && lockfile.strategy && lockfile.strategy != install_config.strategy Log.debug { "Install strategy changed from #{lockfile.strategy} to #{install_config.strategy}" if lockfile.strategy } reporter.info "Install strategy changed from #{lockfile.strategy} to #{install_config.strategy}." if lockfile.strategy @@ -231,7 +247,7 @@ module Zap::Commands::Install config end - private def self.hoisting_check(install_config : Install::Config, lockfile : Lockfile, inferred_context : Zap::Config::InferredContext, reporter : Reporter) + private def self.hoisting_check(install_config : Install::Config, lockfile : Data::Lockfile, inferred_context : Core::Config::InferredContext, reporter : Reporter) : Install::Config if lockfile.update_hoisting_shasum(inferred_context.main_package) if install_config.frozen_lockfile # If the lockfile is frozen, raise an error @@ -241,12 +257,13 @@ module Zap::Commands::Install if lockfile.read_status.from_disk? Log.debug { "Detected a change in hoisting options in the package.json file" } reporter.info("Hoisting options were modified. The packages will be re-installed.") - install_config = install_config.copy_with(refresh_install: true) + return install_config.copy_with(refresh_install: true) end end + install_config end - private def self.package_extensions_check(install_config : Install::Config, lockfile : Lockfile, inferred_context : Zap::Config::InferredContext, reporter : Reporter) + private def self.package_extensions_check(install_config : Install::Config, lockfile : Data::Lockfile, inferred_context : Core::Config::InferredContext, reporter : Reporter) : Install::Config if lockfile.update_package_extensions_shasum(inferred_context.main_package) if install_config.frozen_lockfile # If the lockfile is frozen, raise an error @@ -256,9 +273,10 @@ module Zap::Commands::Install if lockfile.read_status.from_disk? Log.debug { "Detected a change in package extensions options in the package.json file" } reporter.info("Package extensions have been modified. Package metadata will forcefully be fetched from the registry and packages will be re-installed.") - install_config = install_config.copy_with(force_metadata_retrieval: true, refresh_install: true) + return install_config.copy_with(force_metadata_retrieval: true, refresh_install: true) end end + install_config end private def self.remove_packages(state : State) @@ -279,6 +297,7 @@ module Zap::Commands::Install end private def self.resolve_dependencies(state : State) + state.pipeline.set_concurrency(state.config.network_concurrency * 3) state.reporter.report_resolver_updates do # Resolve overrides Log.debug { "• Resolving overrides" } @@ -303,7 +322,7 @@ module Zap::Commands::Install end private def self.resolve_overrides(state : State) - state.lockfile.overrides = Package::Overrides.merge(state.main_package.overrides, state.lockfile.overrides) + state.lockfile.overrides = Data::Package::Overrides.merge(state.main_package.overrides, state.lockfile.overrides) state.lockfile.overrides.try &.each do |name, override_list| override_list.each_with_index do |override, index| Resolver.resolve( @@ -328,8 +347,8 @@ module Zap::Commands::Install pruned_dependencies = state.lockfile.prune(prune_scope) if state.config.global state.install_config.removed_packages.each do |name| - version = Package.get_pkg_version_from_json(Utils::File.join(state.config.node_modules, name, "package.json")) - pruned_dependencies << {name, version, Package::DEFAULT_ROOT} if version + version = Data::Package.get_pkg_version_from_json(Utils::File.join(state.config.node_modules, name, "package.json")) + pruned_dependencies << {name, version, Data::Package::DEFAULT_ROOT} if version end end pruned_dependencies.each do |(name, version)| @@ -364,37 +383,37 @@ module Zap::Commands::Install end end - private def self.install_packages(state : State, pruned_direct_dependencies) - state.reporter.report_installer_updates do - installer = case state.install_config.strategy - when .isolated? - Installer::Isolated.new(state) - when .classic?, .classic_shallow? - Installer::Classic.new(state) - when .pnp? - Installer::PnP.new(state) - else - raise "Unsupported install strategy: #{state.install_config.strategy}" - end + private def self.link_packages(state : State, pruned_direct_dependencies) + state.reporter.report_linker_updates do + linker = case state.install_config.strategy + when .isolated? + Linker::Isolated.new(state) + when .classic?, .classic_shallow? + Linker::Classic.new(state) + when .pnp? + Linker::PnP.new(state) + else + raise "Unsupported install strategy: #{state.install_config.strategy}" + end Log.debug { "• Pruning previous install" } - installer.remove(pruned_direct_dependencies) - installer.prune_orphan_modules + linker.remove(pruned_direct_dependencies) + linker.prune_orphan_modules Log.debug { "• Installing packages" } - installer.install - installer + linker.install + linker end end - private def self.run_install_hooks(state : State, installer : Installer::Base) + private def self.run_install_hooks(state : State, linker : Linker::Base) Log.debug { "• Running install hooks" } - if !state.install_config.ignore_scripts && installer.installed_packages_with_hooks.size > 0 + if !state.install_config.ignore_scripts && linker.installed_packages_with_hooks.size > 0 error_messages = [] of {Exception, String} state.pipeline.reset # Process hooks in parallel state.pipeline.set_concurrency(state.config.concurrency) begin state.reporter.report_builder_updates do - installer.installed_packages_with_hooks.each do |package, path| + linker.installed_packages_with_hooks.each do |package, path| package.scripts.try do |scripts| state.pipeline.process do state.reporter.on_building_package @@ -430,27 +449,27 @@ module Zap::Commands::Install next unless lifecycle_scripts last_script = lifecycle_scripts.pop? next unless last_script - script = Utils::Scripts::ScriptData.new( + script = Data::Package::Scripts::ScriptData.new( package, path, last_script, nil, - before: lifecycle_scripts.map { |s| Utils::Scripts::ScriptDataNested.new(package, path, s, nil) } + before: lifecycle_scripts.map { |s| Data::Package::Scripts::ScriptDataNested.new(package, path, s, nil) } ) }.compact - Utils::Scripts.parallel_run( + Data::Package::Scripts.parallel_run( config: state.config, scripts: scripts, reporter: state.reporter, pipeline: state.pipeline ) - puts NEW_LINE if scripts.size > 0 unless state.config.silent + puts Shared::Constants::NEW_LINE if scripts.size > 0 unless state.config.silent end end - private def self.check_unmet_peer_dependencies(unmet_peers_by_roots : Array(Tuple(Zap::Lockfile::Root, Array(Tuple(String, Zap::Utils::Semver::Range, Zap::Package))))) : Hash(String, Hash(Semver::Range, Set(String))) + private def self.check_unmet_peer_dependencies(unmet_peers_by_roots : Array(Tuple(Data::Lockfile::Root, Array(Tuple(String, Semver::Range, Data::Package))))) : Hash(String, Hash(Semver::Range, Set(String))) # Hash(peer dependency name, Hash(peer dependency version, Set(dependent))) Hash(String, Hash(Semver::Range, Set(String))).new.tap do |unmet_peers| unmet_peers_by_roots.each do |root, unmet_peers_by_root| diff --git a/src/installer/classic/classic.cr b/packages/commands/install/linker/classic/classic.cr similarity index 77% rename from src/installer/classic/classic.cr rename to packages/commands/install/linker/classic/classic.cr index 8dd84c1..6594cc4 100644 --- a/src/installer/classic/classic.cr +++ b/packages/commands/install/linker/classic/classic.cr @@ -1,14 +1,17 @@ -require "../installer" -require "../../backend/*" +require "log" +require "utils/macros" +require "../linker" + +class Commands::Install::Linker::Classic < Commands::Install::Linker::Base + Log = ::Log.for("zap.commands.install.linker.classic") -class Zap::Installer::Classic < Zap::Installer::Base record DependencyItem, # the dependency to install - dependency : Package, + dependency : Data::Package, # a cache of all the possible install locations location_node : LocationNode, # the list of ancestors of this dependency - ancestors : Array(Package), + ancestors : Array(Data::Package), # eventually the name alias alias : String?, # for optional dependencies @@ -28,16 +31,16 @@ class Zap::Installer::Classic < Zap::Installer::Base class Location getter node_modules : Path - getter package : Package - getter hoisted_packages : Hash(String, Package) = Hash(String, Package).new + getter package : Data::Package + getter hoisted_packages : Hash(String, Data::Package) = Hash(String, Data::Package).new getter root : Bool - def initialize(@node_modules : Path, @package : Package, @root : Bool = false) + def initialize(@node_modules : Path, @package : Data::Package, @root : Bool = false) end end class LocationNode < Node(Location) - def self.new(node_modules : Path, package : Package, root : Bool, parent : LocationNode? = nil) + def self.new(node_modules : Path, package : Data::Package, root : Bool, parent : LocationNode? = nil) self.new(Location.new(node_modules, package, root), parent) end @@ -77,8 +80,8 @@ class Zap::Installer::Classic < Zap::Installer::Base dependency_queue << DependencyItem.new( dependency: pkg, location_node: location, - ancestors: workspace ? [workspace.package] : [main_package] of Package, - alias: version_or_alias.is_a?(Package::Alias) ? name : nil, + ancestors: workspace ? [workspace.package] : [main_package] of Data::Package, + alias: version_or_alias.is_a?(Data::Package::Alias) ? name : nil, optional: false ) } @@ -127,7 +130,7 @@ class Zap::Installer::Classic < Zap::Installer::Base dependency: pkg, location_node: install_location.not_nil!, ancestors: ancestors, - alias: version_or_alias.is_a?(Package::Alias) ? name : nil, + alias: version_or_alias.is_a?(Data::Package::Alias) ? name : nil, optional: type.optional_dependency? ) end @@ -137,23 +140,23 @@ class Zap::Installer::Classic < Zap::Installer::Base ancestors_str = dependency_item.ancestors ? dependency_item.ancestors.map { |a| "#{a.name}@#{a.version}" }.join("~>") : "" package_in_error = dependency ? "#{dependency_item.alias.try &.+(":")}#{dependency.name}@#{dependency.version}" : "" state.reporter.error(e, "#{package_in_error.colorize.bold} (#{ancestors_str}) at #{parent_path.colorize.dim}") - exit ErrorCodes::INSTALLER_ERROR.to_i32 + exit Shared::Constants::ErrorCodes::LINKER_ERROR.to_i32 end end end - private def install_dependency(dependency : Package, *, location : LocationNode, ancestors : Array(Package), aliased_name : String?) : Writer::InstallResult + private def install_dependency(dependency : Data::Package, *, location : LocationNode, ancestors : Array(Data::Package), aliased_name : String?) : Writer::InstallResult writer = case dependency.kind in .tarball_file?, .link? - Writer::File.new(dependency, installer: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) + Writer::File.new(dependency, linker: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) in .tarball_url? - Writer::Tarball.new(dependency, installer: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) + Writer::Tarball.new(dependency, linker: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) in .git? - Writer::Git.new(dependency, installer: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) + Writer::Git.new(dependency, linker: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) in .registry? registry_writer = Writer::Registry.new( dependency, - installer: self, + linker: self, location: location, state: state, ancestors: ancestors, @@ -162,17 +165,17 @@ class Zap::Installer::Classic < Zap::Installer::Base return {nil, false} unless registry_writer registry_writer in .workspace? - Writer::Workspace.new(dependency, installer: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) + Writer::Workspace.new(dependency, linker: self, location: location, state: state, ancestors: ancestors, aliased_name: aliased_name) end writer.install end # Actions to perform after the dependency has been freshly installed. - def on_install(dependency : Package, install_folder : Path, *, state : Commands::Install::State, location : LocationNode, ancestors : Array(Package)) + def on_link(dependency : Data::Package, install_folder : Path, *, state : Commands::Install::State, location : LocationNode, ancestors : Array(Data::Package)) # Store package metadata unless File.symlink?(install_folder) - File.open(install_folder / METADATA_FILE_NAME, "w") do |f| + File.open(install_folder / Shared::Constants::METADATA_FILE_NAME, "w") do |f| f.print dependency.key end end @@ -192,7 +195,7 @@ class Zap::Installer::Classic < Zap::Installer::Base if !File.exists?(bin_path) || is_direct_dependency File.delete?(bin_path) File.symlink(Path.new(path).expand(install_folder), bin_path) - File.chmod(bin_path, 0o755) + Utils::Macros.swallow_error { File.chmod(bin_path, 0o755) } end end else @@ -201,14 +204,14 @@ class Zap::Installer::Classic < Zap::Installer::Base if !File.exists?(bin_path) || is_direct_dependency File.delete?(bin_path) File.symlink(Path.new(bin).expand(install_folder), bin_path) - File.chmod(bin_path, 0o755) + Utils::Macros.swallow_error { File.chmod(bin_path, 0o755) } end end end # Copy the scripts from the package.json if dependency.has_install_script - Package.init?(install_folder).try { |pkg| + Data::Package.init?(install_folder).try { |pkg| dependency.scripts = pkg.scripts } end @@ -217,7 +220,7 @@ class Zap::Installer::Classic < Zap::Installer::Base # …npm will default the install command to compile using node-gyp via node-gyp rebuild" # See: https://docs.npmjs.com/cli/v9/using-npm/scripts#npm-install if !dependency.scripts.try &.install && File.exists?(Utils::File.join(install_folder, "binding.gyp")) - (dependency.scripts ||= Zap::Package::LifecycleScripts.new).install = "node-gyp rebuild" + (dependency.scripts ||= Data::Package::LifecycleScripts.new).install = "node-gyp rebuild" end # Register install hook to be executed after the package is installed @@ -227,7 +230,7 @@ class Zap::Installer::Classic < Zap::Installer::Base end # Report that this package has been installed - state.reporter.on_package_installed + state.reporter.on_package_linked end end diff --git a/src/installer/classic/writer/file.cr b/packages/commands/install/linker/classic/writer/file.cr similarity index 79% rename from src/installer/classic/writer/file.cr rename to packages/commands/install/linker/classic/writer/file.cr index c93a270..1ba5a67 100644 --- a/src/installer/classic/writer/file.cr +++ b/packages/commands/install/linker/classic/writer/file.cr @@ -1,17 +1,19 @@ -class Zap::Installer::Classic +require "./writer" + +class Commands::Install::Linker::Classic struct Writer::File < Writer def install : InstallResult case dist = @dependency.dist - when Package::Dist::Link + when Data::Package::Dist::Link install_link(dist) - when Package::Dist::Tarball + when Data::Package::Dist::Tarball install_tarball(dist) else raise "Unknown dist type: #{dist}" end end - def install_link(dist : Package::Dist::Link) : InstallResult + def install_link(dist : Data::Package::Dist::Link) : InstallResult relative_path = dist.link parent = ancestors[0] base_path = state.context.workspaces.try(&.find { |w| w.package == parent }.try &.path) || state.config.prefix @@ -22,19 +24,19 @@ class Zap::Installer::Classic if exists {nil, false} else - state.reporter.on_installing_package + state.reporter.on_linking_package Utils::Directories.mkdir_p(target_path.dirname) FileUtils.rm_rf(target_path) if ::File.directory?(target_path) ::File.symlink(link_source, target_path) - installer.on_install(dependency, target_path, state: state, location: location, ancestors: ancestors) + linker.on_link(dependency, target_path, state: state, location: location, ancestors: ancestors) {nil, true} end end - def install_tarball(dist : Package::Dist::Tarball) : InstallResult + def install_tarball(dist : Data::Package::Dist::Tarball) : InstallResult install_folder = aliased_name || dependency.name target_path = location.value.node_modules / install_folder - exists = Zap::Installer.package_already_installed?(dependency, target_path) + exists = Backend.package_already_installed?(dependency.key, target_path) install_location = self.class.init_location(dependency, target_path, location) if exists @@ -42,7 +44,7 @@ class Zap::Installer::Classic else Utils::Directories.mkdir_p(target_path.dirname) extracted_folder = Path.new(dist.path) - state.reporter.on_installing_package + state.reporter.on_linking_package # TODO :Double check if this is really needed? # @@ -66,7 +68,7 @@ class Zap::Installer::Classic # end FileUtils.cp_r(extracted_folder, target_path) - installer.on_install(dependency, target_path, state: state, location: location, ancestors: ancestors) + linker.on_link(dependency, target_path, state: state, location: location, ancestors: ancestors) {install_location, true} end end diff --git a/src/installer/classic/writer/git.cr b/packages/commands/install/linker/classic/writer/git.cr similarity index 60% rename from src/installer/classic/writer/git.cr rename to packages/commands/install/linker/classic/writer/git.cr index 43bb0f6..ba02765 100644 --- a/src/installer/classic/writer/git.cr +++ b/packages/commands/install/linker/classic/writer/git.cr @@ -1,24 +1,26 @@ -class Zap::Installer::Classic +require "./writer" + +class Commands::Install::Linker::Classic struct Writer::Git < Writer def install : InstallResult - unless packed_tarball_path = dependency.dist.try &.as(Package::Dist::Git).cache_key.try { |key| state.store.package_path(dependency).to_s + ".tgz" } + unless packed_tarball_path = dependency.dist.try &.as(Data::Package::Dist::Git).cache_key.try { |key| state.store.package_path(dependency).to_s + ".tgz" } raise "Cannot install git dependency #{dependency.name} because the dist.cache_key field is missing." end install_folder = aliased_name || dependency.name target_path = location.value.node_modules / install_folder - exists = Zap::Installer.package_already_installed?(dependency, target_path) + exists = Backend.package_already_installed?(dependency.key, target_path) install_location = self.class.init_location(dependency, target_path, location) if exists {install_location, false} else Utils::Directories.mkdir_p(target_path.dirname) - state.reporter.on_installing_package + state.reporter.on_linking_package ::File.open(packed_tarball_path, "r") do |tarball| Utils::TarGzip.unpack_to(tarball, target_path) end - installer.on_install(dependency, target_path, state: state, location: location, ancestors: ancestors) + linker.on_link(dependency, target_path, state: state, location: location, ancestors: ancestors) {install_location, true} end end diff --git a/src/installer/classic/writer/registry.cr b/packages/commands/install/linker/classic/writer/registry.cr similarity index 83% rename from src/installer/classic/writer/registry.cr rename to packages/commands/install/linker/classic/writer/registry.cr index 393cce7..aed0e87 100644 --- a/src/installer/classic/writer/registry.cr +++ b/packages/commands/install/linker/classic/writer/registry.cr @@ -1,4 +1,6 @@ -class Zap::Installer::Classic +require "./writer" + +class Commands::Install::Linker::Classic struct Writer::Registry < Writer def hoist : self? if skip_hoisting? @@ -28,23 +30,23 @@ class Zap::Installer::Classic location: location, ancestors: self.ancestors, aliased_name: self.aliased_name, - installer: self.installer + linker: self.linker ) end def install : InstallResult installation_path = location.value.node_modules / (aliased_name || dependency.name) installed = begin - Backend.install(dependency: dependency, target: installation_path, store: state.store, backend: state.config.file_backend) { - state.reporter.on_installing_package + Backend.link(dependency: dependency, target: installation_path, store: state.store, backend: state.config.file_backend) { + state.reporter.on_linking_package } rescue ex state.reporter.log(%(#{aliased_name.try &.+(":")}#{(dependency.name + '@' + dependency.version).colorize.yellow} Failed to install with #{state.config.file_backend} backend: #{ex.message})) # Fallback to the widely supported "plain copy" backend - Backend.install(backend: :copy, dependency: dependency, target: installation_path, store: state.store) { } + Backend.link(backend: :copy, dependency: dependency, target: installation_path, store: state.store) { } end - installer.on_install(dependency, installation_path, state: state, location: location, ancestors: ancestors) if installed + linker.on_link(dependency, installation_path, state: state, location: location, ancestors: ancestors) if installed {self.class.init_location(dependency, installation_path, location), installed} end @@ -82,14 +84,14 @@ class Zap::Installer::Classic package_dep = package.dependencies.try(&.[dependency.name]?) || package.optional_dependencies.try(&.[dependency.name]?) if package_dep version = package_dep.is_a?(String) ? package_dep : package_dep.version - return HoistAction::Stop unless Utils::Semver.parse(version).satisfies?(dependency.version) + return HoistAction::Stop unless Semver.parse?(version).try &.satisfies?(dependency.version) end # stop hoisting if the package at the current location has a peer dependency but the version of dependency is not compatible package_peer = package.peer_dependencies.try(&.[dependency.name]?) if package_peer version = package_peer.is_a?(String) ? package_peer : package_peer.version - return HoistAction::Stop unless Utils::Semver.parse(version).satisfies?(dependency.version) + return HoistAction::Stop unless Semver.parse?(version).try &.satisfies?(dependency.version) end # stop hoisting if the dependency has a peer dependency on package, no matter the version @@ -100,7 +102,7 @@ class Zap::Installer::Classic # dependency has a peer dependency on a previous hoisted dependency, but the version is not compatible dependency.peer_dependencies.try &.each do |peer_name, peer_version| hoisted = location.value.hoisted_packages[peer_name]? - compatible = !hoisted || Utils::Semver.parse(peer_version).satisfies?(hoisted.version) + compatible = !hoisted || Semver.parse?(peer_version).try &.satisfies?(hoisted.version) return HoistAction::Stop unless compatible end diff --git a/src/installer/classic/writer/tarball.cr b/packages/commands/install/linker/classic/writer/tarball.cr similarity index 57% rename from src/installer/classic/writer/tarball.cr rename to packages/commands/install/linker/classic/writer/tarball.cr index 38c8c82..b1f0423 100644 --- a/src/installer/classic/writer/tarball.cr +++ b/packages/commands/install/linker/classic/writer/tarball.cr @@ -1,20 +1,22 @@ -class Zap::Installer::Classic +require "./writer" + +class Commands::Install::Linker::Classic struct Writer::Tarball < Writer def install : InstallResult install_folder = aliased_name || dependency.name target_path = location.value.node_modules / install_folder - exists = Zap::Installer.package_already_installed?(dependency, target_path) + exists = Backend.package_already_installed?(dependency.key, target_path) install_location = self.class.init_location(dependency, target_path, location) if exists {install_location, false} else Utils::Directories.mkdir_p(target_path.dirname) - extracted_folder = Path.new(dependency.dist.as(Package::Dist::Tarball).path) - state.reporter.on_installing_package + extracted_folder = Path.new(dependency.dist.as(Data::Package::Dist::Tarball).path) + state.reporter.on_linking_package FileUtils.cp_r(extracted_folder, target_path) - installer.on_install(dependency, target_path, state: state, location: location, ancestors: ancestors) + linker.on_link(dependency, target_path, state: state, location: location, ancestors: ancestors) {install_location, true} end end diff --git a/src/installer/classic/writer/workspace.cr b/packages/commands/install/linker/classic/writer/workspace.cr similarity index 71% rename from src/installer/classic/writer/workspace.cr rename to packages/commands/install/linker/classic/writer/workspace.cr index f4c8578..b2f28c8 100644 --- a/src/installer/classic/writer/workspace.cr +++ b/packages/commands/install/linker/classic/writer/workspace.cr @@ -1,7 +1,9 @@ -class Zap::Installer::Classic +require "./writer" + +class Commands::Install::Linker::Classic struct Writer::Workspace < Writer def install : InstallResult - dist = dependency.dist.as(Package::Dist::Workspace) + dist = dependency.dist.as(Data::Package::Dist::Workspace) workspace = state.context.workspaces.not_nil!.find! { |w| w.package.name == dist.workspace } link_source = workspace.path install_folder = aliased_name || dependency.name @@ -10,11 +12,11 @@ class Zap::Installer::Classic if exists {nil, false} else - state.reporter.on_installing_package + state.reporter.on_linking_package Utils::Directories.mkdir_p(target_path.dirname) FileUtils.rm_rf(target_path) if ::File.directory?(target_path) ::File.symlink(link_source, target_path) - installer.on_install(dependency, target_path, state: state, location: location, ancestors: ancestors) + linker.on_link(dependency, target_path, state: state, location: location, ancestors: ancestors) {nil, true} end end diff --git a/packages/commands/install/linker/classic/writer/writer.cr b/packages/commands/install/linker/classic/writer/writer.cr new file mode 100644 index 0000000..36422d4 --- /dev/null +++ b/packages/commands/install/linker/classic/writer/writer.cr @@ -0,0 +1,39 @@ +require "data/package" +require "../classic" + +class Commands::Install::Linker::Classic + abstract struct Writer + getter dependency : Data::Package + getter linker : Linker::Classic + getter location : LocationNode + getter state : Commands::Install::State + getter ancestors : Array(Data::Package) + getter aliased_name : String? + + def initialize( + @dependency : Data::Package, + *, + @linker : Linker::Classic, + @location : LocationNode, + @state : Commands::Install::State, + @ancestors : Array(Data::Package), + @aliased_name : String? + ) + end + + alias InstallResult = {LocationNode?, Bool} + + abstract def install : InstallResult + + def self.init_location(dependency : Data::Package, target_path : Path, location : LocationNode) : LocationNode + LocationNode.new( + node_modules: target_path / "node_modules", + package: dependency, + root: false, + parent: location + ) + end + end +end + +require "./*" diff --git a/src/installer/isolated/isolated.cr b/packages/commands/install/linker/isolated/isolated.cr similarity index 83% rename from src/installer/isolated/isolated.cr rename to packages/commands/install/linker/isolated/isolated.cr index 4b19eed..89a2448 100644 --- a/src/installer/isolated/isolated.cr +++ b/packages/commands/install/linker/isolated/isolated.cr @@ -1,7 +1,12 @@ -require "../installer" -require "../../backend/*" - -class Zap::Installer::Isolated < Zap::Installer::Base +require "log" +require "semver" +require "shared/constants" +require "utils/directories" +require "utils/misc" +require "utils/macros" +require "../linker" + +class Commands::Install::Linker::Isolated < Commands::Install::Linker::Base # See: https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md @node_modules : Path @@ -11,13 +16,13 @@ class Zap::Installer::Isolated < Zap::Installer::Base @public_hoist_patterns : Array(Regex) @installed_packages : Set(String) = Set(String).new - alias Semver = Utils::Semver + Log = ::Log.for("zap.commands.install.linker.isolated") def initialize( state, *, - hoist_patterns = state.main_package.zap_config.try(&.hoist_patterns) || DEFAULT_HOIST_PATTERNS, - public_hoist_patterns = state.main_package.zap_config.try(&.public_hoist_patterns) || DEFAULT_PUBLIC_HOIST_PATTERNS + hoist_patterns = state.main_package.zap_config.try(&.hoist_patterns) || Shared::Constants::DEFAULT_HOIST_PATTERNS, + public_hoist_patterns = state.main_package.zap_config.try(&.public_hoist_patterns) || Shared::Constants::DEFAULT_PUBLIC_HOIST_PATTERNS ) super(state) @node_modules = Path.new(state.config.node_modules) @@ -27,8 +32,8 @@ class Zap::Installer::Isolated < Zap::Installer::Base @hoisted_store = @modules_store / "node_modules" Utils::Directories.mkdir_p(@hoisted_store) - @hoist_patterns = hoist_patterns.map &->Utils::Various.parse_pattern(String) - @public_hoist_patterns = public_hoist_patterns.map &->Utils::Various.parse_pattern(String) + @hoist_patterns = hoist_patterns.map &->Utils::Misc.parse_pattern(String) + @public_hoist_patterns = public_hoist_patterns.map &->Utils::Misc.parse_pattern(String) end def install : Nil @@ -52,7 +57,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base end private def install_package( - package : Package | Lockfile::Root, + package : Data::Package | Data::Lockfile::Root, *, ancestors : Ancestors, root_path : Path? = nil, @@ -61,7 +66,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base resolved_peers = nil overrides = nil - if package.is_a?(Package) + if package.is_a?(Data::Package) Log.debug { "(#{package.key}) Installing package…" } # Raise if the architecture is not supported - unless the package is optional @@ -71,7 +76,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base if package.kind.link? root = ancestors.last base_path = state.context.workspaces.try(&.find { |w| w.package.name == root.name }.try &.path) || state.config.prefix - return Path.new(package.dist.as(Package::Dist::Link).link).expand(base_path) + return Path.new(package.dist.as(Data::Package::Dist::Link).link).expand(base_path) elsif package.kind.workspace? workspace = state.context.workspaces.not_nil!.find! { |w| w.package.name == package.name } return Path.new(workspace.path) @@ -83,7 +88,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base package_folder = String.build do |str| str << package.hashed_key if resolved_peers && resolved_peers.size > 0 - peers_hash = Package.hash_dependencies(resolved_peers) + peers_hash = Data::Package.hash_dependencies(resolved_peers) str << "+#{peers_hash}" end if resolved_transitive_overrides && resolved_transitive_overrides.size > 0 @@ -101,7 +106,6 @@ class Zap::Installer::Isolated < Zap::Installer::Base end # If the package folder exists, we assume that the package dependencies were already installed too - package_path = install_path / package.name if File.directory?(install_path) # If there is no need to perform a full pass, we can just return the package path and skip the dependencies unless state.install_config.refresh_install @@ -121,13 +125,13 @@ class Zap::Installer::Isolated < Zap::Installer::Base Utils::Directories.mkdir_p(install_path) case package.kind when .tarball_file? - Writer::File.install(package, package_path, installer: self, state: state) + Writer::File.install(package, package_path, linker: self, state: state) when .tarball_url? - Writer::Tarball.install(package, package_path, installer: self, state: state) + Writer::Tarball.install(package, package_path, linker: self, state: state) when .git? - Writer::Git.install(package, package_path, installer: self, state: state) + Writer::Git.install(package, package_path, linker: self, state: state) when .registry? - Writer::Registry.install(package, package_path, installer: self, state: state) + Writer::Registry.install(package, package_path, linker: self, state: state) end end else @@ -151,11 +155,11 @@ class Zap::Installer::Isolated < Zap::Installer::Base # For each resolved peer and pinned dependency, install the dependency in the .store folder if it's not already installed if resolved_peers - pinned_packages + resolved_peers.map { |p| {p.name, p, Package::DependencyType::Dependency} } + pinned_packages + resolved_peers.map { |p| {p.name, p, Data::Package::DependencyType::Dependency} } else pinned_packages end.each do |(name, dependency, type)| - Log.debug { "(#{package.is_a?(Package) ? package.key : package.name}) Processing dependency: #{dependency.key}" } + Log.debug { "(#{package.is_a?(Data::Package) ? package.key : package.name}) Processing dependency: #{dependency.key}" } # Add to the ancestors ancestors.unshift(package) @@ -175,31 +179,31 @@ class Zap::Installer::Isolated < Zap::Installer::Base # Link it to the parent package target = install_path / name - Log.debug { "(#{package.is_a?(Package) ? package.key : package.name}) Linking #{dependency.key}: #{source} -> #{target}" } + Log.debug { "(#{package.is_a?(Data::Package) ? package.key : package.name}) Linking #{dependency.key}: #{source} -> #{target}" } symlink(source, target) # Link binaries link_binaries(dependency, package_path: target, target_node_modules: install_path) end - if package.is_a?(Package) + if package.is_a?(Data::Package) return install_path / package.name else return install_path end end - def on_install(dependency : Package, install_folder : Path, *, state : Commands::Install::State) + def on_link(dependency : Data::Package, install_folder : Path, *, state : Commands::Install::State) # Store package metadata unless File.symlink?(install_folder) - File.open(install_folder / METADATA_FILE_NAME, "w") do |f| + File.open(install_folder / Shared::Constants::METADATA_FILE_NAME, "w") do |f| f.print dependency.key end end # Copy the scripts from the package.json if dependency.has_install_script - Package.init?(install_folder).try do |pkg| + Data::Package.init?(install_folder).try do |pkg| dependency.scripts = pkg.scripts end end @@ -208,7 +212,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base # …npm will default the install command to compile using node-gyp via node-gyp rebuild" # See: https://docs.npmjs.com/cli/v9/using-npm/scripts#npm-install if !dependency.scripts.try &.install && File.exists?(Utils::File.join(install_folder, "binding.gyp")) - (dependency.scripts ||= Zap::Package::LifecycleScripts.new).install = "node-gyp rebuild" + (dependency.scripts ||= Data::Package::LifecycleScripts.new).install = "node-gyp rebuild" end # Register install hook to be executed after the package is installed @@ -221,7 +225,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base hoist_package(dependency, install_folder) # Report the package as installed - state.reporter.on_package_installed + state.reporter.on_package_linked end def prune_orphan_modules @@ -232,7 +236,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base prune_workspace_orphans(@hoisted_store, unlink_binaries?: false) end - protected def link_binaries(package : Package, *, package_path : Path, target_node_modules : Path) + protected def link_binaries(package : Data::Package, *, package_path : Path, target_node_modules : Path) if bin = package.bin base_bin_path = target_node_modules / ".bin" Utils::Directories.mkdir_p(base_bin_path) @@ -242,14 +246,14 @@ class Zap::Installer::Isolated < Zap::Installer::Base bin_path = Utils::File.join(base_bin_path, bin_name) File.delete?(bin_path) File.symlink(Path.new(path).expand(package_path), bin_path) - File.chmod(bin_path, 0o755) + Utils::Macros.swallow_error { File.chmod(bin_path, 0o755) } end else bin_name = package.name.split("/").last bin_path = Utils::File.join(base_bin_path, bin_name) File.delete?(bin_path) File.symlink(Path.new(bin).expand(package_path), bin_path) - File.chmod(bin_path, 0o755) + Utils::Macros.swallow_error { File.chmod(bin_path, 0o755) } end end end @@ -281,7 +285,7 @@ class Zap::Installer::Isolated < Zap::Installer::Base end end - private def hoist_package(package : Package, install_folder : Path) + private def hoist_package(package : Data::Package, install_folder : Path) if @public_hoist_patterns.any?(&.=~ package.name) # Hoist to the root node_modules folder unless state.main_package.has_dependency?(package.name) diff --git a/packages/commands/install/linker/isolated/writer/file.cr b/packages/commands/install/linker/isolated/writer/file.cr new file mode 100644 index 0000000..b66c0b8 --- /dev/null +++ b/packages/commands/install/linker/isolated/writer/file.cr @@ -0,0 +1,17 @@ +module Commands::Install::Linker::Isolated::Writer::File + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + case dist = dependency.dist + when Data::Package::Dist::Tarball + exists = Backend.package_already_installed?(dependency.key, install_path) + unless exists + extracted_folder = Path.new(dist.path) + state.reporter.on_linking_package + + FileUtils.cp_r(extracted_folder, install_path) + linker.on_link(dependency, install_path, state: state) + end + else + raise "Unknown dist type: #{dist}" + end + end +end diff --git a/packages/commands/install/linker/isolated/writer/git.cr b/packages/commands/install/linker/isolated/writer/git.cr new file mode 100644 index 0000000..29c08bb --- /dev/null +++ b/packages/commands/install/linker/isolated/writer/git.cr @@ -0,0 +1,17 @@ +module Commands::Install::Linker::Isolated::Writer::Git + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + unless packed_tarball_path = dependency.dist.try &.as(Data::Package::Dist::Git).cache_key.try { |key| state.store.package_path(dependency).to_s + ".tgz" } + raise "Cannot install git dependency #{dependency.name} because the dist.cache_key field is missing." + end + + exists = Backend.package_already_installed?(dependency.key, install_path) + + unless exists + state.reporter.on_linking_package + ::File.open(packed_tarball_path, "r") do |tarball| + Utils::TarGzip.unpack_to(tarball, install_path) + end + linker.on_link(dependency, install_path, state: state) + end + end +end diff --git a/packages/commands/install/linker/isolated/writer/registry.cr b/packages/commands/install/linker/isolated/writer/registry.cr new file mode 100644 index 0000000..98dfbbe --- /dev/null +++ b/packages/commands/install/linker/isolated/writer/registry.cr @@ -0,0 +1,15 @@ +module Commands::Install::Linker::Isolated::Writer::Registry + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + installed = begin + Backend.link(dependency: dependency, target: install_path, store: state.store, backend: state.config.file_backend) { + state.reporter.on_linking_package + } + rescue ex + state.reporter.log(%(#{("#{dependency.name}@#{dependency.version}").colorize.yellow} Failed to install with #{state.config.file_backend} backend: #{ex.message})) + # Fallback to the widely supported "plain copy" backend + Backend.link(backend: :copy, dependency: dependency, target: install_path, store: state.store) { } + end + + linker.on_link(dependency, install_path, state: state) if installed + end +end diff --git a/packages/commands/install/linker/isolated/writer/tarball.cr b/packages/commands/install/linker/isolated/writer/tarball.cr new file mode 100644 index 0000000..2fa1478 --- /dev/null +++ b/packages/commands/install/linker/isolated/writer/tarball.cr @@ -0,0 +1,17 @@ +module Commands::Install::Linker::Isolated::Writer::Tarball + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + case dist = dependency.dist + when Data::Package::Dist::Tarball + exists = Backend.package_already_installed?(dependency.key, install_path) + unless exists + extracted_folder = Path.new(dist.path) + state.reporter.on_linking_package + + FileUtils.cp_r(extracted_folder, install_path) + linker.on_link(dependency, install_path, state: state) + end + else + raise "Unknown dist type: #{dist}" + end + end +end diff --git a/src/installer/installer.cr b/packages/commands/install/linker/linker.cr similarity index 79% rename from src/installer/installer.cr rename to packages/commands/install/linker/linker.cr index 0e38cf9..9aa857c 100644 --- a/src/installer/installer.cr +++ b/packages/commands/install/linker/linker.cr @@ -1,32 +1,16 @@ -module Zap::Installer - METADATA_FILE_NAME = ".zap.metadata" - - Log = Zap::Log.for(self) - - # Check if a package is already installed on the filesystem - def self.package_already_installed?(dependency : Package, path : Path) - if exists = Dir.exists?(path) - metadata_path = path / METADATA_FILE_NAME - unless File.readable?(metadata_path) - FileUtils.rm_rf(path) - exists = false - else - key = File.read(metadata_path) - if key != dependency.key - FileUtils.rm_rf(path) - exists = false - end - end - end - exists - end +require "log" +require "data/package" +require "../state" + +module Commands::Install::Linker + Log = ::Log.for("zap.commands.install.linker") abstract class Base getter state : Commands::Install::State - getter main_package : Package - getter installed_packages_with_hooks = [] of {Package, Path} + getter main_package : Data::Package + getter installed_packages_with_hooks = [] of {Data::Package, Path} - alias Ancestors = Deque(Package | Lockfile::Root) + alias Ancestors = Deque(Data::Package | Data::Lockfile::Root) def initialize(state : Commands::Install::State) @state = state @@ -36,13 +20,13 @@ module Zap::Installer abstract def install : Nil # Remove a set of direct dependencies from the filesystem - def remove(dependencies : Set({String, String | Package::Alias, String})) : Nil + def remove(dependencies : Set({String, String | Data::Package::Alias, String})) : Nil dependencies.each do |(name, version_or_alias, root_name)| workspace = state.context.workspaces.try &.find { |w| w.package.name == root_name } node_modules = workspace.try(&.path./ "node_modules") || Path.new(state.config.node_modules) package_path = node_modules / name if File.directory?(package_path) - package = Package.init?(package_path) + package = Data::Package.init?(package_path) version = version_or_alias.is_a?(String) ? version_or_alias : version_or_alias.version if package && package.name == name && package.version == version unlink_binaries(package, package_path) @@ -73,7 +57,7 @@ module Zap::Installer if should_prune_orphan?(package_path) Log.debug { "Pruning orphan package: #{package_dir}" } if unlink_binaries? - package = Package.init?(package_path) + package = Data::Package.init?(package_path) if package unlink_binaries(package, package_path) end @@ -92,7 +76,7 @@ module Zap::Installer private def should_prune_orphan?(package_path : Path) : Bool remove_child = false - metadata_path = package_path / METADATA_FILE_NAME + metadata_path = package_path / Shared::Constants::METADATA_FILE_NAME if File.readable?(metadata_path) # Check the metadata file to retrieve the package key key = File.read(metadata_path) @@ -105,7 +89,7 @@ module Zap::Installer remove_child end - protected def unlink_binaries(package : Package, package_path : Path) + protected def unlink_binaries(package : Data::Package, package_path : Path) if bin = package.bin if bin.is_a?(Hash) bin.each do |name, path| @@ -122,12 +106,14 @@ module Zap::Installer end # Resolve which peer dependencies should be available to a package given its ancestors - protected def resolve_peers(package : Package, ancestors : Ancestors) : Set(Package)? + protected def resolve_peers(package : Data::Package, ancestors : Ancestors) : Set(Data::Package)? # Aggregate direct and transitive peer dependencies peers = Hash(String, Set(Semver::Range)).new if direct_peers = package.peer_dependencies direct_peers.each do |direct_peer, peer_range| - peers[direct_peer] = Set(Semver::Range){Semver.parse(peer_range)} + peers[direct_peer] = Set(Semver::Range){ + Semver.parse?(peer_range).or(Semver::ANY), + } end end if transitive_peers = package.transitive_peer_dependencies @@ -142,7 +128,7 @@ module Zap::Installer # If there are any peers, resolve them if peers.size > 0 number_of_peers = peers.reduce(0) { |acc, (k, v)| acc + v.size } - Set(Package).new.tap do |resolved_peers| + Set(Data::Package).new.tap do |resolved_peers| # For each ancestor, check if it has a pinned dependency that matches the peer ancestors.each do |ancestor| ancestor.each_dependency do |name, version_or_alias| @@ -166,8 +152,8 @@ module Zap::Installer end end - protected def resolve_transitive_overrides(package : Package, ancestors : Ancestors) : Set(Package::Overrides::Parent)? - result = Set(Package::Overrides::Parent).new + protected def resolve_transitive_overrides(package : Data::Package, ancestors : Ancestors) : Set(Data::Package::Overrides::Parent)? + result = Set(Data::Package::Overrides::Parent).new if transitive_overrides = package.transitive_overrides reversed_ancestors = ancestors.to_a.reverse transitive_overrides.each do |override| @@ -184,11 +170,11 @@ module Zap::Installer # Otherwise return the original package. protected def apply_override( state : Commands::Install::State, - package : Package, - ancestors : Enumerable(Package | Lockfile::Root), + package : Data::Package, + ancestors : Enumerable(Data::Package | Data::Lockfile::Root), *, reverse_ancestors? : Bool = false - ) : Package + ) : Data::Package if overrides = state.lockfile.overrides ancestors = ancestors.to_a.reverse if reverse_ancestors? if override = overrides.override?(package, ancestors) @@ -196,7 +182,7 @@ module Zap::Installer # ancestors_str = ancestors.map { |a| "#{a.name}@#{a.version}" }.join(" > ") # state.reporter.log("#{"Overriden:".colorize.bold.yellow} #{"#{override.name}@"}#{override.specifier.colorize.blue} (was: #{package.version}) #{"(#{ancestors_str})".colorize.dim}") Log.debug { - ancestors_str = ancestors.select(&.is_a?(Package)).map { |a| "#{a.as(Package).name}@#{a.as(Package).version}" }.join(" > ") + ancestors_str = ancestors.select(&.is_a?(Data::Package)).map { |a| "#{a.as(Data::Package).name}@#{a.as(Data::Package).version}" }.join(" > ") "(#{package.key}) Overriden dependency: #{"#{override.name}@"}#{override.specifier} (was: #{package.version}) (#{ancestors_str})" } return state.lockfile.packages["#{override.name}@#{override.specifier}"] diff --git a/src/installer/pnp/manifest.cr b/packages/commands/install/linker/pnp/manifest.cr similarity index 98% rename from src/installer/pnp/manifest.cr rename to packages/commands/install/linker/pnp/manifest.cr index a22285c..527be42 100644 --- a/src/installer/pnp/manifest.cr +++ b/packages/commands/install/linker/pnp/manifest.cr @@ -1,7 +1,7 @@ require "json" # See: https://yarnpkg.com/advanced/pnp-spec#manifest-reference -struct Zap::Installer::PnP::Manifest +struct Commands::Install::Linker::PnP::Manifest include JSON::Serializable getter __info : Array(String) = [ diff --git a/src/installer/pnp/pnp.cr b/packages/commands/install/linker/pnp/pnp.cr similarity index 84% rename from src/installer/pnp/pnp.cr rename to packages/commands/install/linker/pnp/pnp.cr index 472257e..674eabf 100644 --- a/src/installer/pnp/pnp.cr +++ b/packages/commands/install/linker/pnp/pnp.cr @@ -1,13 +1,17 @@ -require "../installer" +require "log" +require "utils/macros" +require "../linker" # See: https://yarnpkg.com/advanced/pnp-spec -class Zap::Installer::PnP < Zap::Installer::Base +class Commands::Install::Linker::PnP < Commands::Install::Linker::Base @installed_packages : Set(String) = Set(String).new @node_modules : Path @modules_store : Path @relative_modules_store : Path @manifest : Manifest = Manifest.new + Log = ::Log.for("zap.commands.install.linker.pnp") + def initialize(state : Commands::Install::State) super(state) @node_modules = Path.new(state.config.node_modules) @@ -16,7 +20,7 @@ class Zap::Installer::PnP < Zap::Installer::Base Utils::Directories.mkdir_p(@modules_store) end - alias Ancestors = Deque(Package | Lockfile::Root) + alias Ancestors = Deque(Data::Package | Data::Lockfile::Root) alias PackageReference = String | {String, String} def install : Nil @@ -50,18 +54,18 @@ class Zap::Installer::PnP < Zap::Installer::Base end private def install_package( - package_or_root : Package | Lockfile::Root, + package_or_root : Data::Package | Data::Lockfile::Root, *, ancestors : Ancestors, optional : Bool = false, - workspace_or_main_package : (Workspaces::Workspace | Package)? = nil + workspace_or_main_package : (Workspaces::Workspace | Data::Package)? = nil ) : {reference: PackageReference, path: (Path | String)?}? resolved_peers = nil overrides = nil root = ancestors.last? - main_package = workspace_or_main_package.is_a?(Package) ? workspace_or_main_package : nil + main_package = workspace_or_main_package.is_a?(Data::Package) ? workspace_or_main_package : nil - if package_or_root.is_a?(Package) + if package_or_root.is_a?(Data::Package) package = package_or_root name = package.name reference = package.key @@ -73,7 +77,7 @@ class Zap::Installer::PnP < Zap::Installer::Base # Shortcut for links, no need to check its dependencies if package.kind.link? - location = "./#{Path.posix(package.dist.as(Package::Dist::Link).link).relative_to(state.config.prefix)}/" + location = "./#{Path.posix(package.dist.as(Data::Package::Dist::Link).link).relative_to(state.config.prefix)}/" @manifest.package_registry_data.add_package_data( name, reference, @@ -92,7 +96,7 @@ class Zap::Installer::PnP < Zap::Installer::Base # Check the satisfied peer dependencies + overrides then derive a unique hash to identify the virtualized package maybe_virtualized_hashes = String.build do |str| if resolved_peers && resolved_peers.size > 0 - peers_hash = Package.hash_dependencies(resolved_peers) + peers_hash = Data::Package.hash_dependencies(resolved_peers) str << "#{peers_hash}" end if resolved_transitive_overrides && resolved_transitive_overrides.size > 0 @@ -136,13 +140,13 @@ class Zap::Installer::PnP < Zap::Installer::Base Utils::Directories.mkdir_p(install_path) case package.kind when .tarball_file? - Writer::File.install(package, install_path, installer: self, state: state) + Writer::File.install(package, install_path, linker: self, state: state) when .tarball_url? - Writer::Tarball.install(package, install_path, installer: self, state: state) + Writer::Tarball.install(package, install_path, linker: self, state: state) when .git? - Writer::Git.install(package, install_path, installer: self, state: state) + Writer::Git.install(package, install_path, linker: self, state: state) when .registry? - Writer::Registry.install(package, install_path, installer: self, state: state) + Writer::Registry.install(package, install_path, linker: self, state: state) end else # Prevents infinite loops and duplicate checks @@ -213,11 +217,11 @@ class Zap::Installer::PnP < Zap::Installer::Base # For each resolved peer and pinned dependency, install the dependency and link it to the parent package through the manifest data if resolved_peers - pinned_packages + resolved_peers.map { |p| {p.name, p, Package::DependencyType::Dependency} } + pinned_packages + resolved_peers.map { |p| {p.name, p, Data::Package::DependencyType::Dependency} } else pinned_packages end.each do |(name, dependency, type)| - Log.debug { "(#{package_or_root.is_a?(Package) ? package_or_root.key : package_or_root.name}) Processing dependency: #{dependency.key}" } + Log.debug { "(#{package_or_root.is_a?(Data::Package) ? package_or_root.key : package_or_root.name}) Processing dependency: #{dependency.key}" } # Add to the ancestors ancestors.unshift(package_or_root) @@ -238,7 +242,7 @@ class Zap::Installer::PnP < Zap::Installer::Base dep_path = install_data[:path] # Link it to the parent package - Log.debug { "(#{package_or_root.is_a?(Package) ? package_or_root.key : package_or_root.name}) Linking #{dependency.key}: #{dep_reference} -> #{reference}" } + Log.debug { "(#{package_or_root.is_a?(Data::Package) ? package_or_root.key : package_or_root.name}) Linking #{dependency.key}: #{dep_reference} -> #{reference}" } package_dependencies << Manifest::Data::PackageDependency.new( name: name, reference: name != dependency.name ? {dependency.name, dep_reference} : dep_reference @@ -267,17 +271,17 @@ class Zap::Installer::PnP < Zap::Installer::Base {reference: reference, path: install_path} end - def on_install(dependency : Package, install_folder : Path, *, state : Commands::Install::State) + def on_link(dependency : Data::Package, install_folder : Path, *, state : Commands::Install::State) # Store package metadata unless File.symlink?(install_folder) - File.open(install_folder / METADATA_FILE_NAME, "w") do |f| + File.open(install_folder / Shared::Constants::METADATA_FILE_NAME, "w") do |f| f.print dependency.key end end # Copy the scripts from the package.json if dependency.has_install_script - Package.init?(install_folder).try do |pkg| + Data::Package.init?(install_folder).try do |pkg| dependency.scripts = pkg.scripts end end @@ -286,7 +290,7 @@ class Zap::Installer::PnP < Zap::Installer::Base # …npm will default the install command to compile using node-gyp via node-gyp rebuild" # See: https://docs.npmjs.com/cli/v9/using-npm/scripts#npm-install if !dependency.scripts.try &.install && File.exists?(Utils::File.join(install_folder, "binding.gyp")) - (dependency.scripts ||= Zap::Package::LifecycleScripts.new).install = "node-gyp rebuild" + (dependency.scripts ||= Data::Package::LifecycleScripts.new).install = "node-gyp rebuild" end # Register install hook to be executed after the package is installed @@ -296,14 +300,14 @@ class Zap::Installer::PnP < Zap::Installer::Base end # Report the package as installed - state.reporter.on_package_installed + state.reporter.on_package_linked end def prune_orphan_modules prune_workspace_orphans(@modules_store, unlink_binaries?: true) end - protected def link_binaries(package : Package, package_path : Path, target : Path) + protected def link_binaries(package : Data::Package, package_path : Path, target : Path) if bin = package.bin target_bin_path = target / "node_modules" / ".bin" Utils::Directories.mkdir_p(target_bin_path) @@ -313,14 +317,14 @@ class Zap::Installer::PnP < Zap::Installer::Base bin_path = Utils::File.join(target_bin_path, bin_name) File.delete?(bin_path) File.symlink(Path.new(path).expand(package_path), bin_path) - File.chmod(bin_path, 0o755) + Utils::Macros.swallow_error { File.chmod(bin_path, 0o755) } end else bin_name = package.name.split("/").last bin_path = Utils::File.join(target_bin_path, bin_name) File.delete?(bin_path) File.symlink(Path.new(bin).expand(package_path), bin_path) - File.chmod(bin_path, 0o755) + Utils::Macros.swallow_error { File.chmod(bin_path, 0o755) } end end end diff --git a/src/installer/pnp/runtime/.pnp.cjs b/packages/commands/install/linker/pnp/runtime/.pnp.cjs similarity index 100% rename from src/installer/pnp/runtime/.pnp.cjs rename to packages/commands/install/linker/pnp/runtime/.pnp.cjs diff --git a/src/installer/pnp/runtime/.pnp.loader.mjs b/packages/commands/install/linker/pnp/runtime/.pnp.loader.mjs similarity index 100% rename from src/installer/pnp/runtime/.pnp.loader.mjs rename to packages/commands/install/linker/pnp/runtime/.pnp.loader.mjs diff --git a/src/installer/pnp/runtime/read_runtime.cr b/packages/commands/install/linker/pnp/runtime/read_runtime.cr similarity index 100% rename from src/installer/pnp/runtime/read_runtime.cr rename to packages/commands/install/linker/pnp/runtime/read_runtime.cr diff --git a/src/installer/pnp/runtime/runtime.cr b/packages/commands/install/linker/pnp/runtime/runtime.cr similarity index 97% rename from src/installer/pnp/runtime/runtime.cr rename to packages/commands/install/linker/pnp/runtime/runtime.cr index 8aa9a2d..cb047a0 100644 --- a/src/installer/pnp/runtime/runtime.cr +++ b/packages/commands/install/linker/pnp/runtime/runtime.cr @@ -1,4 +1,4 @@ -module Zap::Installer::PnP::Runtime +module Commands::Install::Linker::PnP::Runtime # All credits to the Yarn team for the runtime code. ❤️ # See: https://github.com/yarnpkg/berry/tree/master/packages/yarnpkg-pnp # diff --git a/packages/commands/install/linker/pnp/writer/file.cr b/packages/commands/install/linker/pnp/writer/file.cr new file mode 100644 index 0000000..ba6dbaa --- /dev/null +++ b/packages/commands/install/linker/pnp/writer/file.cr @@ -0,0 +1,17 @@ +module Commands::Install::Linker::PnP::Writer::File + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + case dist = dependency.dist + when Data::Package::Dist::Tarball + exists = Backend.package_already_installed?(dependency.key, install_path) + unless exists + extracted_folder = Path.new(dist.path) + state.reporter.on_linking_package + + FileUtils.cp_r(extracted_folder, install_path) + linker.on_link(dependency, install_path, state: state) + end + else + raise "Unknown dist type: #{dist}" + end + end +end diff --git a/packages/commands/install/linker/pnp/writer/git.cr b/packages/commands/install/linker/pnp/writer/git.cr new file mode 100644 index 0000000..40f00fe --- /dev/null +++ b/packages/commands/install/linker/pnp/writer/git.cr @@ -0,0 +1,17 @@ +module Commands::Install::Linker::PnP::Writer::Git + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + unless packed_tarball_path = dependency.dist.try &.as(Data::Package::Dist::Git).cache_key.try { |key| state.store.package_path(dependency).to_s + ".tgz" } + raise "Cannot install git dependency #{dependency.name} because the dist.cache_key field is missing." + end + + exists = Backend.package_already_installed?(dependency.key, install_path) + + unless exists + state.reporter.on_linking_package + ::File.open(packed_tarball_path, "r") do |tarball| + Utils::TarGzip.unpack_to(tarball, install_path) + end + linker.on_link(dependency, install_path, state: state) + end + end +end diff --git a/packages/commands/install/linker/pnp/writer/registry.cr b/packages/commands/install/linker/pnp/writer/registry.cr new file mode 100644 index 0000000..fb8a090 --- /dev/null +++ b/packages/commands/install/linker/pnp/writer/registry.cr @@ -0,0 +1,15 @@ +module Commands::Install::Linker::PnP::Writer::Registry + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + installed = begin + Backend.link(dependency: dependency, target: install_path, store: state.store, backend: state.config.file_backend) { + state.reporter.on_linking_package + } + rescue ex + state.reporter.log(%(#{("#{dependency.name}@#{dependency.version}").colorize.yellow} Failed to install with #{state.config.file_backend} backend: #{ex.message})) + # Fallback to the widely supported "plain copy" backend + Backend.link(backend: :copy, dependency: dependency, target: install_path, store: state.store) { } + end + + linker.on_link(dependency, install_path, state: state) if installed + end +end diff --git a/packages/commands/install/linker/pnp/writer/tarball.cr b/packages/commands/install/linker/pnp/writer/tarball.cr new file mode 100644 index 0000000..b7826a0 --- /dev/null +++ b/packages/commands/install/linker/pnp/writer/tarball.cr @@ -0,0 +1,17 @@ +module Commands::Install::Linker::PnP::Writer::Tarball + def self.install(dependency : Data::Package, install_path : Path, *, linker : Linker::Base, state : Commands::Install::State) + case dist = dependency.dist + when Data::Package::Dist::Tarball + exists = Backend.package_already_installed?(dependency.key, install_path) + unless exists + extracted_folder = Path.new(dist.path) + state.reporter.on_linking_package + + FileUtils.cp_r(extracted_folder, install_path) + linker.on_link(dependency, install_path, state: state) + end + else + raise "Unknown dist type: #{dist}" + end + end +end diff --git a/src/commands/install/manifest.cr b/packages/commands/install/manifest.cr similarity index 91% rename from src/commands/install/manifest.cr rename to packages/commands/install/manifest.cr index d52b498..d5c50c6 100644 --- a/src/commands/install/manifest.cr +++ b/packages/commands/install/manifest.cr @@ -1,18 +1,16 @@ require "json" require "msgpack" -require "../../utils/semver" +require "semver" -struct Zap::Commands::Install::Manifest +struct Commands::Install::Manifest include JSON::Serializable include MessagePack::Serializable - alias Semver = Utils::Semver - getter dist_tags : Hash(String, String) = Hash(String, String).new getter versions_json : Hash(String, String) = Hash(String, String).new getter versions : Array(String) = Array(String).new - def initialize(manifest_string : String) + def initialize(manifest_string : String | IO) @dist_tags = Hash(String, String).new @versions_json = Hash(String, String).new @versions = Array(String).new @@ -52,7 +50,7 @@ struct Zap::Commands::Install::Manifest @versions.sort! { |a, b| Semver::Version.parse(b) <=> Semver::Version.parse(a) } end - def get_raw_metadata?(version : Utils::Semver::Range | String) : String? + def get_raw_metadata?(version : Semver::Range | String) : String? raw_metadata = nil case version @@ -60,7 +58,7 @@ struct Zap::Commands::Install::Manifest # Find the version that matches the dist-tag tag_version = dist_tags[version]? raw_metadata = @versions_json[tag_version]? if tag_version - in Utils::Semver::Range + in Semver::Range if version.exact_match? # For exact comparisons - we compare the version string raw_metadata = @versions_json[version.to_s]? diff --git a/src/commands/install/protocol/alias/alias.cr b/packages/commands/install/protocol/alias/alias.cr similarity index 79% rename from src/commands/install/protocol/alias/alias.cr rename to packages/commands/install/protocol/alias/alias.cr index df1ad5a..471b198 100644 --- a/src/commands/install/protocol/alias/alias.cr +++ b/packages/commands/install/protocol/alias/alias.cr @@ -1,8 +1,13 @@ +require "log" +require "utils/misc" +require "extensions/object" require "../../resolver" require "../base" require "../registry" -struct Zap::Commands::Install::Protocol::Alias < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::Alias < Commands::Install::Protocol::Base + Log = ::Log.for("zap.commands.install.protocol.alias") + def self.normalize?(str : String, path_info : PathInfo?) : {String?, String?}? if (parts = str.split("@npm:")).size > 1 # @npm: @@ -31,11 +36,11 @@ struct Zap::Commands::Install::Protocol::Alias < Zap::Commands::Install::Protoco return nil unless stripped_specifier - package_name, package_specifier = Utils::Various.parse_key(stripped_specifier) + package_name, package_specifier = Utils::Misc.parse_key(stripped_specifier) final_specifier = package_specifier.pipe { |s| if s.nil? "latest" - elsif semver = Utils::Semver.parse?(s) + elsif semver = Semver.parse?(s) semver else Log.debug { "(#{name}@#{specifier}) Failed to parse semver '#{s}', treating as a dist-tag." } diff --git a/packages/commands/install/protocol/aliased.cr b/packages/commands/install/protocol/aliased.cr new file mode 100644 index 0000000..77434d7 --- /dev/null +++ b/packages/commands/install/protocol/aliased.cr @@ -0,0 +1,7 @@ +module Commands::Install::Protocol + record Aliased, name : String, alias : String do + def to_s(io) + io << "#{name} (alias:#{@alias})" + end + end +end diff --git a/src/commands/install/protocol/base.cr b/packages/commands/install/protocol/base.cr similarity index 88% rename from src/commands/install/protocol/base.cr rename to packages/commands/install/protocol/base.cr index b62ce66..2656be9 100644 --- a/src/commands/install/protocol/base.cr +++ b/packages/commands/install/protocol/base.cr @@ -1,12 +1,6 @@ require "./resolver" -module Zap::Commands::Install::Protocol - record Aliased, name : String, alias : String do - def to_s(io) - io << "#{name} (alias:#{@alias})" - end - end - +module Commands::Install::Protocol record PathInfo, base_directory : String, path : Path do def self.from_str(str : String, base_directory : String) : PathInfo? unless str.starts_with?(".") || str.starts_with?("/") || str.starts_with?("~") diff --git a/src/commands/install/protocol/file/file.cr b/packages/commands/install/protocol/file/file.cr similarity index 86% rename from src/commands/install/protocol/file/file.cr rename to packages/commands/install/protocol/file/file.cr index 96104e6..cee5c3a 100644 --- a/src/commands/install/protocol/file/file.cr +++ b/packages/commands/install/protocol/file/file.cr @@ -1,8 +1,11 @@ +require "log" require "../../resolver" require "../base" require "./resolver" -struct Zap::Commands::Install::Protocol::File < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::File < Commands::Install::Protocol::Base + Log = ::Log.for("zap.commands.install.protocol.file") + def self.normalize?(str : String, path_info : PathInfo?) : {String?, String?}? return nil unless path_info path_str = path_info.path.to_s diff --git a/src/commands/install/protocol/file/resolver.cr b/packages/commands/install/protocol/file/resolver.cr similarity index 69% rename from src/commands/install/protocol/file/resolver.cr rename to packages/commands/install/protocol/file/resolver.cr index 13889d2..bae4740 100644 --- a/src/commands/install/protocol/file/resolver.cr +++ b/packages/commands/install/protocol/file/resolver.cr @@ -2,14 +2,14 @@ require "../../resolver" require "../base" require "../resolver" -struct Zap::Commands::Install::Protocol::File < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::File < Commands::Install::Protocol::Base end -struct Zap::Commands::Install::Protocol::File::Resolver < Zap::Commands::Install::Protocol::Resolver - def resolve(*, pinned_version : String? = nil) : Package +struct Commands::Install::Protocol::File::Resolver < Commands::Install::Protocol::Resolver + def resolve(*, pinned_version : String? = nil) : Data::Package path = Path.new @specifier[5..] base_path = begin - if parent.is_a?(Lockfile::Root) + if parent.is_a?(Data::Lockfile::Root) state.context.workspaces.try(&.find { |w| w.package.name == parent.not_nil!.name }.try &.path) || state.config.prefix else nil @@ -18,8 +18,8 @@ struct Zap::Commands::Install::Protocol::File::Resolver < Zap::Commands::Install raise "the file: protocol is forbidden for non-direct dependencies" if base_path.nil? absolute_path = path.expand(base_path) if Dir.exists? absolute_path - Package.init(absolute_path).tap { |pkg| - pkg.dist = Package::Dist::Link.new(path.to_s) + Data::Package.init(absolute_path).tap { |pkg| + pkg.dist = Data::Package::Dist::Link.new(path.to_s) on_resolve(pkg) } elsif ::File.exists? absolute_path @@ -30,8 +30,8 @@ struct Zap::Commands::Install::Protocol::File::Resolver < Zap::Commands::Install store_hash = Digest::SHA1.hexdigest("#{tarball_path}") + "-#{checksum}" temp_path = Path.new(Dir.tempdir, store_hash) self.class.extract_tarball_to_temp(absolute_path, temp_path) - Package.init(temp_path).tap { |pkg| - pkg.dist = Package::Dist::Tarball.new(tarball_path.to_s, temp_path.to_s) + Data::Package.init(temp_path).tap { |pkg| + pkg.dist = Data::Package::Dist::Tarball.new(tarball_path.to_s, temp_path.to_s) on_resolve(pkg) } else @@ -39,12 +39,12 @@ struct Zap::Commands::Install::Protocol::File::Resolver < Zap::Commands::Install end end - def valid?(metadata : Package) : Bool + def valid?(metadata : Data::Package) : Bool false end - def store?(metadata : Package, &on_downloading) : Bool - if (dist = metadata.dist).is_a?(Package::Dist::Tarball) + def store?(metadata : Data::Package, &on_downloading) : Bool + if (dist = metadata.dist).is_a?(Data::Package::Dist::Tarball) self.class.extract_tarball_to_temp(dist.tarball, Path.new(dist.path)) end false diff --git a/src/commands/install/protocol/git/git.cr b/packages/commands/install/protocol/git/git.cr similarity index 82% rename from src/commands/install/protocol/git/git.cr rename to packages/commands/install/protocol/git/git.cr index fcac506..c7991be 100644 --- a/src/commands/install/protocol/git/git.cr +++ b/packages/commands/install/protocol/git/git.cr @@ -1,10 +1,13 @@ +require "log" require "../base" require "./resolver" -require "../../../../constants" -require "../../../../utils/concurrent/dedupe_lock" +require "shared/constants" +require "concurrency/dedupe_lock" -struct Zap::Commands::Install::Protocol::Git < Zap::Commands::Install::Protocol::Base - Utils::Concurrent::DedupeLock::Global.setup(:clone, Package) +struct Commands::Install::Protocol::Git < Commands::Install::Protocol::Base + Log = ::Log.for("zap.commands.install.protocol.git") + + Concurrency::DedupeLock::Global.setup(:clone, Data::Package) def self.normalize?(str : String, path_info : PathInfo?) : {String?, String?}? if str.starts_with?("github:") @@ -19,7 +22,7 @@ struct Zap::Commands::Install::Protocol::Git < Zap::Commands::Install::Protocol: elsif str.starts_with?("gitlab:") # gitlab:/[#] return "git+https://gitlab.com/#{str[7..]}", nil - elsif str.starts_with?("git+") || str.starts_with?("git://") || str.matches?(GH_SHORT_REGEX) + elsif str.starts_with?("git+") || str.starts_with?("git://") || str.matches?(Shared::Constants::GH_SHORT_REGEX) # # /[#] return str, nil @@ -43,7 +46,7 @@ struct Zap::Commands::Install::Protocol::Git < Zap::Commands::Install::Protocol: when .starts_with?("github:") Log.debug { "(#{name}@#{specifier}) Resolved as a github dependency" } Resolver::Github.new(state, name, specifier[7..], parent, dependency_type, skip_cache) - when .matches?(GH_SHORT_REGEX) + when .matches?(Shared::Constants::GH_SHORT_REGEX) Log.debug { "(#{name}@#{specifier}) Resolved as a github dependency" } Resolver::Github.new(state, name, specifier, parent, dependency_type, skip_cache) end diff --git a/src/commands/install/protocol/git/resolver.cr b/packages/commands/install/protocol/git/resolver.cr similarity index 77% rename from src/commands/install/protocol/git/resolver.cr rename to packages/commands/install/protocol/git/resolver.cr index 1b91d9a..16647d2 100644 --- a/src/commands/install/protocol/git/resolver.cr +++ b/packages/commands/install/protocol/git/resolver.cr @@ -1,13 +1,15 @@ +require "git/remote" +require "data/package" +require "reporter/proxy" require "../base" require "../resolver" -require "../../../../utils/git/remote" -struct Zap::Commands::Install::Protocol::Git < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::Git < Commands::Install::Protocol::Base end -module Zap::Commands::Install::Protocol::Git::Resolver - private abstract struct Base < Zap::Commands::Install::Protocol::Resolver - getter git_remote : Utils::Git::Remote +module Commands::Install::Protocol::Git::Resolver + private abstract struct Base < Commands::Install::Protocol::Resolver + getter git_remote : ::Git::Remote def initialize( state, @@ -18,20 +20,20 @@ module Zap::Commands::Install::Protocol::Git::Resolver skip_cache = false ) super - @git_remote = Utils::Git::Remote.new(@specifier.to_s, @state.reporter) + @git_remote = ::Git::Remote.new(@specifier.to_s, Reporter::ReporterPrependPipe.new(@state.reporter)) end - def resolve(*, pinned_version : String? = nil) : Package + def resolve(*, pinned_version : String? = nil) : Data::Package fetch_metadata.tap do |pkg| on_resolve(pkg) end end - def valid?(metadata : Package) : Bool - !!metadata.dist.as?(Package::Dist::Git).try(&.version.== specifier.to_s) + def valid?(metadata : Data::Package) : Bool + !!metadata.dist.as?(Data::Package::Dist::Git).try(&.version.== specifier.to_s) end - def fetch_metadata : Package + def fetch_metadata : Data::Package commit_hash = @git_remote.commitish_hash cache_key = Digest::SHA1.hexdigest("#{@git_remote.short_key}") metadata_cache_key = "#{@name}__git:#{cache_key}.package.json" @@ -43,16 +45,16 @@ module Zap::Commands::Install::Protocol::Git::Resolver metadata_path = @name.nil? ? nil : @state.store.file_path(metadata_cache_key) metadata_cached = metadata_path && ::File.exists?(metadata_path) clone_to(cloned_repo_path) unless cloned || metadata_cached - Package.init(metadata_cached && metadata_path ? metadata_path : cloned_repo_path, append_filename: !metadata_cached).tap do |pkg| + Data::Package.init(metadata_cached && metadata_path ? metadata_path : cloned_repo_path, append_filename: !metadata_cached).tap do |pkg| @state.store.store_file(metadata_cache_key, pkg.to_json) unless metadata_cached - pkg.dist = Package::Dist::Git.new(commit_hash, specifier.to_s, @git_remote.key, cache_key) + pkg.dist = Data::Package::Dist::Git.new(commit_hash, specifier.to_s, @git_remote.key, cache_key) end end end end - def store?(metadata : Package, &on_downloading) : Bool - cache_key = metadata.dist.as(Package::Dist::Git).cache_key + def store?(metadata : Data::Package, &on_downloading) : Bool + cache_key = metadata.dist.as(Data::Package::Dist::Git).cache_key cloned_folder_path = Path.new(Dir.tempdir, cache_key) tarball_path = Path.new(@state.store.package_path(metadata).to_s + ".tgz") packed = ::File.exists?(tarball_path) @@ -94,13 +96,14 @@ module Zap::Commands::Install::Protocol::Git::Resolver private def prepare_package( path : Path, - config : ::Zap::Config = @state.config.copy_for_inner_consumption.copy_with(prefix: path.to_s) + config : ::Core::Config = @state.config.copy_for_inner_consumption.copy_with(prefix: path.to_s) ) Commands::Install.run( config, Commands::Install::Config.new(save: false), store: @state.store, - raise_on_failure: true + raise_on_failure: true, + reporter: Reporter::Proxy.new(@state.reporter), ) end diff --git a/src/commands/install/protocol/protocol.cr b/packages/commands/install/protocol/protocol.cr similarity index 75% rename from src/commands/install/protocol/protocol.cr rename to packages/commands/install/protocol/protocol.cr index c6f6f26..692a9c2 100644 --- a/src/commands/install/protocol/protocol.cr +++ b/packages/commands/install/protocol/protocol.cr @@ -1,4 +1,4 @@ -require "../../../log" +require "log" require "./alias" require "./file" require "./git" @@ -6,8 +6,8 @@ require "./registry" require "./tarball_url" require "./workspace" -module Zap::Commands::Install::Protocol - Log = Zap::Log.for(self) +module Commands::Install::Protocol + Log = ::Log.for("zap.install.protocol") PROTOCOLS = { Protocol::Workspace, diff --git a/src/commands/install/protocol/registry/registry.cr b/packages/commands/install/protocol/registry/registry.cr similarity index 81% rename from src/commands/install/protocol/registry/registry.cr rename to packages/commands/install/protocol/registry/registry.cr index f127043..40b5f82 100644 --- a/src/commands/install/protocol/registry/registry.cr +++ b/packages/commands/install/protocol/registry/registry.cr @@ -1,7 +1,10 @@ +require "log" require "../base" require "./resolver" -struct Zap::Commands::Install::Protocol::Registry < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::Registry < Commands::Install::Protocol::Base + Log = ::Log.for("zap.commands.install.protocol.registry") + # [<@scope>/] # [<@scope>/]@ # [<@scope>/]@ @@ -23,7 +26,7 @@ struct Zap::Commands::Install::Protocol::Registry < Zap::Commands::Install::Prot skip_cache = false ) : Protocol::Resolver? Log.debug { "(#{name}@#{specifier}) Resolved as a registry dependency" } - semver = Utils::Semver.parse?(specifier) + semver = Semver.parse?(specifier) Log.debug { "(#{name}@#{specifier}) Failed to parse semver '#{specifier}', treating as a dist-tag." } unless semver Resolver.new(state, name, semver || specifier, parent, dependency_type, skip_cache) end diff --git a/src/commands/install/protocol/registry/resolver.cr b/packages/commands/install/protocol/registry/resolver.cr similarity index 68% rename from src/commands/install/protocol/registry/resolver.cr rename to packages/commands/install/protocol/registry/resolver.cr index d7b6b8f..e52adfb 100644 --- a/src/commands/install/protocol/registry/resolver.cr +++ b/packages/commands/install/protocol/registry/resolver.cr @@ -1,15 +1,19 @@ +require "log" +require "fetch" +require "shared/constants" require "../base" require "../resolver" require "../../manifest" require "../../registry_clients" -require "../../../../utils/fetch" -struct Zap::Commands::Install::Protocol::Registry < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::Registry < Commands::Install::Protocol::Base end -struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Install::Protocol::Resolver +struct Commands::Install::Protocol::Registry::Resolver < Commands::Install::Protocol::Resolver + Log = ::Log.for("zap.commands.install.protocol.registry.resolver") + @clients : RegistryClients - @client_pool : Utils::Fetch(Manifest) + @client_pool : Fetch(Manifest) @package_name : String getter base_url : URI @@ -19,7 +23,7 @@ struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Ins specifier = "latest", parent = nil, dependency_type = nil, - skip_cache = false + skip_cache = false, ) super @@ -40,36 +44,36 @@ struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Ins @client_pool = @clients.get_or_init_pool(@base_url.to_s) end - def resolve(*, pinned_version : String? = nil) : Package + def resolve(*, pinned_version : String? = nil) : Data::Package pkg = fetch_metadata(pinned_version: pinned_version) on_resolve(pkg) pkg rescue e - Zap::Log.debug { e.message.colorize.red.to_s + NEW_LINE + e.backtrace.map { |line| "\t#{line}" }.join(NEW_LINE).colorize.red.to_s } + Log.debug { e.message.colorize.red.to_s + Shared::Constants::NEW_LINE + e.backtrace.map { |line| "\t#{line}" }.join(Shared::Constants::NEW_LINE).colorize.red.to_s } raise "Error resolving #{pkg.try &.name || self.name} #{pkg.try &.version || self.specifier} #{e.message}" end - def valid?(metadata : Package) : Bool + def valid?(metadata : Data::Package) : Bool range_set = self.specifier metadata.kind.registry? && ( (range_set.is_a?(String) && range_set == metadata.version) || - (range_set.is_a?(Utils::Semver::Range) && range_set.satisfies?(metadata.version)) + (range_set.is_a?(Semver::Range) && range_set.satisfies?(metadata.version)) ) end - def store?(metadata : Package, &on_downloading) : Bool + def store?(metadata : Data::Package, &on_downloading) : Bool state.store.with_lock(metadata, state.config) do next false if state.store.package_is_cached?(metadata) yield - dist = metadata.dist.not_nil!.as(Package::Dist::Registry) + dist = metadata.dist.not_nil!.as(Data::Package::Dist::Registry) tarball_url = dist.tarball integrity = dist.integrity.try &.split(" ")[0] shasum = dist.shasum version = metadata.version unsupported_algorithm = false - algorithm, hash, algorithm_instance = nil, nil, nil + algorithm, hash = nil, nil if integrity algorithm, hash = integrity.split("-") @@ -77,17 +81,18 @@ struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Ins unsupported_algorithm = true end - algorithm_instance = case algorithm - when "sha1" - Digest::SHA1.new - when "sha256" - Digest::SHA256.new - when "sha512" - Digest::SHA512.new - else - unsupported_algorithm = true - Digest::SHA1.new - end + algorithm_instance = + case algorithm + when "sha1" + -> { Digest::SHA1.new } + when "sha256" + -> { Digest::SHA256.new } + when "sha512" + -> { Digest::SHA512.new } + else + unsupported_algorithm = true + -> { Digest::SHA1.new } + end # the tarball_url is absolute and can point to an entirely different domain # so we need to find the right client pool for it @@ -99,7 +104,7 @@ struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Ins client.get("/" + relative_url) do |response| raise "Invalid status code from #{tarball_url} (#{response.status_code})" unless response.status_code == 200 - IO::Digest.new(response.body_io, algorithm_instance).tap do |io| + IO::Digest.new(response.body_io, algorithm_instance.call).tap do |io| state.store.unpack_and_store_tarball(metadata, io) io.skip_to_end @@ -115,7 +120,7 @@ struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Ins end rescue e state.store.remove_package(metadata) - raise e + raise Exception.new("Unable to download package #{metadata.name}@#{metadata.version} from #{tarball_url}: #{e.message}", e) end true end @@ -123,19 +128,19 @@ struct Zap::Commands::Install::Protocol::Registry::Resolver < Zap::Commands::Ins end end - private def fetch_metadata(*, pinned_version : String? = nil) : Package? + private def fetch_metadata(*, pinned_version : String? = nil) : Data::Package? Log.debug { "(#{@name}@#{specifier}) Fetching metadata… #{@skip_cache ? "(skipping cache)" : ""} #{pinned_version ? "[pinned_version #{pinned_version}]" : ""}" } state.store.with_lock("#{@base_url.to_s}/#{@package_name}", state.config) do metadata_url = @base_url.relativize("/#{@package_name}").to_s manifest = @skip_cache ? @client_pool.client { |http| - Manifest.new(http.get(metadata_url, HEADERS).body) - } : @client_pool.fetch_with_cache(metadata_url, HEADERS) { |body| Manifest.new(body) } + Manifest.new(http.get(metadata_url, Shared::Constants::HEADERS).body) + } : @client_pool.fetch_with_cache(metadata_url, Shared::Constants::HEADERS) { |body| Manifest.new(body) } Log.debug { "(#{@name}@#{@specifier}) Checking the registry metadata for a match against the version/dist-tag" } - raw_metadata = manifest.get_raw_metadata?(pinned_version ? Utils::Semver.parse(pinned_version) : self.specifier) + raw_metadata = manifest.get_raw_metadata?(pinned_version ? Semver.parse(pinned_version) : self.specifier) unless raw_metadata raise "No version matching range or dist-tag #{specifier} for package #{@name} found in the module registry" end - Package.from_json(raw_metadata) + Data::Package.from_json(raw_metadata) end end end diff --git a/src/commands/install/protocol/resolver.cr b/packages/commands/install/protocol/resolver.cr similarity index 65% rename from src/commands/install/protocol/resolver.cr rename to packages/commands/install/protocol/resolver.cr index 5add918..1d87831 100644 --- a/src/commands/install/protocol/resolver.cr +++ b/packages/commands/install/protocol/resolver.cr @@ -1,22 +1,25 @@ -require "../../../utils/semver" -require "../../../utils/concurrent/dedupe_lock" -require "../../../utils/concurrent/keyed_lock" -require "../../../commands/install/state" -require "../../../package" -require "../../../lockfile" +require "semver" +require "concurrency/dedupe_lock" +require "concurrency/keyed_lock" +require "data/package" +require "data/lockfile" +require "./aliased" +require "../state" -abstract struct Zap::Commands::Install::Protocol::Resolver - alias Specifier = String | Utils::Semver::Range +abstract struct Commands::Install::Protocol::Resolver + Log = ::Log.for("zap.commands.install.protocol.resolver") + + alias Specifier = String | Semver::Range getter name : (String | Aliased)? getter specifier : Specifier getter state : Commands::Install::State - getter parent : (Package | Lockfile::Root)? = nil - getter dependency_type : Package::DependencyType? = nil + getter parent : (Data::Package | Data::Lockfile::Root)? = nil + getter dependency_type : Data::Package::DependencyType? = nil getter skip_cache : Bool = false - Utils::Concurrent::DedupeLock::Global.setup(:store, Bool) - Utils::Concurrent::KeyedLock::Global.setup(Package) + Concurrency::DedupeLock::Global.setup(:store, Bool) + Concurrency::KeyedLock::Global.setup(Data::Package) def initialize( @state, @@ -28,14 +31,14 @@ abstract struct Zap::Commands::Install::Protocol::Resolver ) end - abstract def resolve(*, pinned_version : String? = nil) : Package - abstract def valid?(metadata : Package) : Bool - abstract def store?(metadata : Package, &on_downloading) : Bool + abstract def resolve(*, pinned_version : String? = nil) : Data::Package + abstract def valid?(metadata : Data::Package) : Bool + abstract def store?(metadata : Data::Package, &on_downloading) : Bool - protected def on_resolve(pkg : Package) + protected def on_resolve(pkg : Data::Package) aliased_name = @name.pipe { |name| name.is_a?(Aliased) ? name.alias : nil } parent_package = parent - if parent_package.is_a?(Lockfile::Root) + if parent_package.is_a?(Data::Lockfile::Root) # For direct dependencies: check if the package is freshly added since the last install and report accordingly if parent_specifier = parent_package.dependency_specifier?(aliased_name || pkg.name) if pkg.specifier != parent_specifier @@ -53,9 +56,9 @@ abstract struct Zap::Commands::Install::Protocol::Resolver pkg.has_prepare_script ||= pkg.scripts.try(&.has_prepare_script?) # Pin the dependency to the locked version if aliased_name - parent_package.try &.dependency_specifier(aliased_name, Package::Alias.new(name: pkg.name, version: pkg.specifier), @dependency_type) + parent_package.try &.dependency_specifier(aliased_name, Data::Package::Alias.new(name: pkg.name, version: pkg.specifier), @dependency_type) else - Log.debug { "Setting dependency specifier for #{pkg.name} to #{pkg.specifier} in #{parent_package.specifier}" } if parent_package.is_a?(Package) + Log.debug { "Setting dependency specifier for #{pkg.name} to #{pkg.specifier} in #{parent_package.specifier}" } if parent_package.is_a?(Data::Package) parent_package.try &.dependency_specifier(pkg.name, pkg.specifier, @dependency_type) end end @@ -64,7 +67,7 @@ abstract struct Zap::Commands::Install::Protocol::Resolver parent_package = @parent pinned_dependency = parent_package.try &.dependency_specifier?(name) if pinned_dependency - if pinned_dependency.is_a?(Package::Alias) + if pinned_dependency.is_a?(Data::Package::Alias) packages_ref = pinned_dependency.key else packages_ref = "#{name}@#{pinned_dependency}" diff --git a/packages/commands/install/protocol/tarball_url/resolver.cr b/packages/commands/install/protocol/tarball_url/resolver.cr new file mode 100644 index 0000000..0c53b59 --- /dev/null +++ b/packages/commands/install/protocol/tarball_url/resolver.cr @@ -0,0 +1,33 @@ +require "../base" +require "../resolver" +require "utils/targzip" + +struct Commands::Install::Protocol::TarballUrl < Commands::Install::Protocol::Base +end + +struct Commands::Install::Protocol::TarballUrl::Resolver < Commands::Install::Protocol::Resolver + def resolve(*, pinned_version : String? = nil) : Data::Package + tarball_url = specifier.to_s + @state.store.with_lock(tarball_url, @state.config) do + temp_path = @state.store.store_temp_tarball(tarball_url) + Data::Package.init(temp_path).tap { |pkg| + pkg.dist = Data::Package::Dist::Tarball.new(tarball_url, temp_path.to_s) + on_resolve(pkg) + } + end + end + + def valid?(metadata : Data::Package) : Bool + false + end + + def store?(metadata : Data::Package, &on_downloading) : Bool + dist = metadata.dist.as(Data::Package::Dist::Tarball) + return false if Dir.exists?(dist.path) + yield + state.store.with_lock(dist.tarball, state.config) do + Utils::TarGzip.download_and_unpack(dist.tarball, Path.new(dist.path)) + end + true + end +end diff --git a/src/commands/install/protocol/tarball_url/tarball_url.cr b/packages/commands/install/protocol/tarball_url/tarball_url.cr similarity index 76% rename from src/commands/install/protocol/tarball_url/tarball_url.cr rename to packages/commands/install/protocol/tarball_url/tarball_url.cr index d0d840f..822e148 100644 --- a/src/commands/install/protocol/tarball_url/tarball_url.cr +++ b/packages/commands/install/protocol/tarball_url/tarball_url.cr @@ -1,7 +1,10 @@ +require "log" require "../base" require "./resolver" -struct Zap::Commands::Install::Protocol::TarballUrl < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::TarballUrl < Commands::Install::Protocol::Base + Log = ::Log.for("zap.commands.install.protocol.tarball_url") + def self.normalize?(str : String, path_info : PathInfo?) : {String?, String?}? # if str.starts_with?("https://") || str.starts_with?("http://") @@ -16,7 +19,7 @@ struct Zap::Commands::Install::Protocol::TarballUrl < Zap::Commands::Install::Pr parent = nil, dependency_type = nil, skip_cache = false - ) : Zap::Commands::Install::Protocol::Resolver? + ) : Commands::Install::Protocol::Resolver? if specifier.starts_with?("http://") || specifier.starts_with?("https://") Log.debug { "(#{name}@#{specifier}) Resolved as a tarball url dependency" } Resolver.new(state, name, specifier, parent, dependency_type, skip_cache) diff --git a/src/commands/install/protocol/workspace/resolver.cr b/packages/commands/install/protocol/workspace/resolver.cr similarity index 52% rename from src/commands/install/protocol/workspace/resolver.cr rename to packages/commands/install/protocol/workspace/resolver.cr index efe1810..ddbfba7 100644 --- a/src/commands/install/protocol/workspace/resolver.cr +++ b/packages/commands/install/protocol/workspace/resolver.cr @@ -1,10 +1,10 @@ require "../base" require "../resolver" -struct Zap::Commands::Install::Protocol::Workspace < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::Workspace < Commands::Install::Protocol::Base end -struct Zap::Commands::Install::Protocol::Workspace::Resolver < Zap::Commands::Install::Protocol::Resolver +struct Commands::Install::Protocol::Workspace::Resolver < Commands::Install::Protocol::Resolver @workspace : Workspaces::Workspace def initialize( @@ -19,10 +19,10 @@ struct Zap::Commands::Install::Protocol::Workspace::Resolver < Zap::Commands::In super(state, name, specifier, parent, dependency_type, skip_cache) end - def resolve(*, pinned_version : String? = nil) : Package + def resolve(*, pinned_version : String? = nil) : Data::Package if Dir.exists?(@workspace.path) - Package.init(@workspace.path).tap { |pkg| - pkg.dist = Package::Dist::Workspace.new(@workspace.package.name) + Data::Package.init(@workspace.path).tap { |pkg| + pkg.dist = Data::Package::Dist::Workspace.new(@workspace.package.name) on_resolve(pkg) } else @@ -30,11 +30,11 @@ struct Zap::Commands::Install::Protocol::Workspace::Resolver < Zap::Commands::In end end - def valid?(metadata : Package) : Bool + def valid?(metadata : Data::Package) : Bool false end - def store?(metadata : Package, &on_downloading) : Bool + def store?(metadata : Data::Package, &on_downloading) : Bool false end end diff --git a/src/commands/install/protocol/workspace/workspace.cr b/packages/commands/install/protocol/workspace/workspace.cr similarity index 88% rename from src/commands/install/protocol/workspace/workspace.cr rename to packages/commands/install/protocol/workspace/workspace.cr index d117ba1..86e435a 100644 --- a/src/commands/install/protocol/workspace/workspace.cr +++ b/packages/commands/install/protocol/workspace/workspace.cr @@ -1,7 +1,10 @@ +require "log" require "../base" require "./resolver" -struct Zap::Commands::Install::Protocol::Workspace < Zap::Commands::Install::Protocol::Base +struct Commands::Install::Protocol::Workspace < Commands::Install::Protocol::Base + Log = ::Log.for("zap.commands.install.protocol.workspace") + def self.normalize?(str : String, path_info : PathInfo?) : {String?, String?}? nil end @@ -21,7 +24,7 @@ struct Zap::Commands::Install::Protocol::Workspace < Zap::Commands::Install::Pro return nil if name.nil? # Check if the package depending on the current one is a root package - parent_is_workspace = !parent || parent.is_a?(Lockfile::Root) + parent_is_workspace = !parent || parent.is_a?(Data::Lockfile::Root) workspaces = state.context.workspaces is_workspace_protocol = specifier.starts_with?("workspace:") diff --git a/src/commands/install/registry_clients.cr b/packages/commands/install/registry_clients.cr similarity index 66% rename from src/commands/install/registry_clients.cr rename to packages/commands/install/registry_clients.cr index 0fc945b..cfec19d 100644 --- a/src/commands/install/registry_clients.cr +++ b/packages/commands/install/registry_clients.cr @@ -1,14 +1,14 @@ require "./manifest" -require "../../utils/fetch" +require "fetch" # Exposes a pool of http(s) clients for each registry and convenience methods to access the pools. # # The pools are lazily initialized and cached. -class Zap::Commands::Install::RegistryClients +class Commands::Install::RegistryClients # The pool of clients for each registry - @client_pool_by_registry : Hash(String, Utils::Fetch(Manifest)) = Hash(String, Utils::Fetch(Manifest)).new + @@client_pool_by_registry : Hash(String, Fetch(Manifest)) = Hash(String, Fetch(Manifest)).new # Lock to synchronize access to the pools - @client_pool_by_registry_lock = Mutex.new + @@client_pool_by_registry_lock = Mutex.new # Initialize a new registry clients pool with the following arguments: # - store_path: path to the store where the metadata will be cached @@ -17,7 +17,7 @@ class Zap::Commands::Install::RegistryClients # - bypass_staleness_checks: whether to bypass the staleness checks when reading from the cache def initialize( @store_path : String, - @npmrc : Npmrc, + @npmrc : Data::Npmrc, *, @pool_max_size : Int32 = 20, @bypass_staleness_checks : Bool = false @@ -25,28 +25,30 @@ class Zap::Commands::Install::RegistryClients end # Returns the client pool for the given registry url or creates a new one if it doesn't exist. - def get_or_init_pool(url : String) : Utils::Fetch(Manifest) - @client_pool_by_registry_lock.synchronize do - @client_pool_by_registry[url] ||= init_client_pool(url) + def get_or_init_pool(url : String) : Fetch(Manifest) + @@client_pool_by_registry_lock.synchronize do + @@client_pool_by_registry[url] ||= init_client_pool(url) end end # Attempts to find a matching client pool for the given (tarball) url or creates a new one if it doesn't exist. # Strips the path from the url before matching. - def find_or_init_pool(url : String) : {String, Utils::Fetch(Manifest)} - # Find if an existing pool matches the url - @client_pool_by_registry.find do |registry_url, _| - url.starts_with?(registry_url) - end || begin - # Otherwise create a new pool for the url hostname - uri = URI.parse(url) - # Remove the path - because it is impossible to infer based on the tarball url - uri.path = "/" - uri_str = uri.to_s - pool = init_client_pool(uri.to_s).tap do |pool| - @client_pool_by_registry[pool.base_url] = pool + def find_or_init_pool(url : String) : {String, Fetch(Manifest)} + @@client_pool_by_registry_lock.synchronize do + # Find if an existing pool matches the url + @@client_pool_by_registry.find do |registry_url, _| + url.starts_with?(registry_url) + end || begin + # Otherwise create a new pool for the url hostname + uri = URI.parse(url) + # Remove the path - because it is impossible to infer based on the tarball url + uri.path = "/" + uri_str = uri.to_s + pool = init_client_pool(uri.to_s).tap do |pool| + @@client_pool_by_registry[pool.base_url] = pool + end + {uri_str, pool} end - {uri_str, pool} end end @@ -56,18 +58,18 @@ class Zap::Commands::Install::RegistryClients *, pool_max_size = @pool_max_size, bypass_staleness_checks = @bypass_staleness_checks - ) : Utils::Fetch(Manifest) + ) : Fetch(Manifest) # Cache the metadata in the store - filesystem_cache = Utils::Fetch::Cache::InStore(Manifest).new( + filesystem_cache = Fetch::Cache::InStore(Manifest).new( @store_path, bypass_staleness_checks: bypass_staleness_checks, - serializer: Utils::Fetch::Cache::InStore::MessagePackSerializer(Manifest).new + serializer: Fetch::Cache::InStore::MessagePackSerializer(Manifest).new ) - # memory_cache = Utils::Fetch::Cache::InMemory(Manifest).new(fallback: filesystem_cache) + # memory_cache = Fetch::Cache::InMemory(Manifest).new(fallback: filesystem_cache) authentication = @npmrc.registries_auth[base_url]? - Utils::Fetch.new( + Fetch.new( base_url, pool_max_size: pool_max_size, # cache: memory_cache diff --git a/src/commands/install/resolver.cr b/packages/commands/install/resolver.cr similarity index 84% rename from src/commands/install/resolver.cr rename to packages/commands/install/resolver.cr index 18844d8..a86db46 100644 --- a/src/commands/install/resolver.cr +++ b/packages/commands/install/resolver.cr @@ -1,29 +1,30 @@ -require "../../constants" -require "../../utils/semver" -require "../../utils/data_structures/safe_set" -require "../../utils/concurrent/keyed_lock" -require "../../utils/concurrent/dedupe_lock" -require "../../utils/concurrent/pipeline" -require "../../store" +require "log" +require "shared/constants" +require "semver" +require "concurrency/data_structures/safe_set" +require "concurrency/keyed_lock" +require "concurrency/dedupe_lock" +require "concurrency/pipeline" +require "store" require "./state" require "./protocol/resolver" require "./protocol" -module Zap::Commands::Install::Resolver - Log = Zap::Log.for(self) +module Commands::Install::Resolver + Log = ::Log.for("zap.commands.install.resolver") - alias Pipeline = ::Zap::Utils::Concurrent::Pipeline + alias Pipeline = ::Concurrency::Pipeline - Utils::Concurrent::DedupeLock::Global.setup(:store, Bool) - Utils::Concurrent::KeyedLock::Global.setup(Package) + Concurrency::DedupeLock::Global.setup(:store, Bool) + Concurrency::KeyedLock::Global.setup(Data::Package) def self.get( state : Commands::Install::State, name : String?, specifier : String = "latest", - parent : Package | Lockfile::Root | Nil = nil, - dependency_type : Package::DependencyType? = nil, - skip_cache : Bool = false + parent : Data::Package | Data::Lockfile::Root | Nil = nil, + dependency_type : Data::Package::DependencyType? = nil, + skip_cache : Bool = false, ) : Protocol::Resolver resolver = Protocol::PROTOCOLS.reduce(nil) do |acc, protocol| next acc unless acc.nil? @@ -40,12 +41,12 @@ module Zap::Commands::Install::Resolver end def self.resolve_dependencies_of( - package : Package, + package : Data::Package, *, state : Commands::Install::State, - ancestors : Deque(Package) = Deque(Package).new, + ancestors : Deque(Data::Package) = Deque(Data::Package).new, disable_cache_for_packages : Array(String)? = nil, - disable_cache : Bool = false + disable_cache : Bool = false, ) is_root = ancestors.size == 0 package.each_dependency( @@ -67,7 +68,7 @@ module Zap::Commands::Install::Resolver end || false end) - if version_or_alias.is_a?(Package::Alias) + if version_or_alias.is_a?(Data::Package::Alias) version = version_or_alias.to_s else version = version_or_alias @@ -80,23 +81,23 @@ module Zap::Commands::Install::Resolver type, state: state, is_direct_dependency: is_root, - ancestors: Deque(Package).new(ancestors.size + 1).concat(ancestors).push(package), + ancestors: Deque(Data::Package).new(ancestors.size + 1).concat(ancestors).push(package), bust_pinned_cache: bust_pinned_cache ) end end def self.resolve( - package : Package?, + package : Data::Package?, name : String, version : String, - type : Package::DependencyType? = nil, + type : Data::Package::DependencyType? = nil, *, state : Commands::Install::State, is_direct_dependency : Bool = false, single_resolution : Bool = false, - ancestors : Deque(Package) = Deque(Package).new, - bust_pinned_cache : Bool = false + ancestors : Deque(Data::Package) = Deque(Data::Package).new, + bust_pinned_cache : Bool = false, ) resolve( package, @@ -112,17 +113,17 @@ module Zap::Commands::Install::Resolver end def self.resolve( - package : Package?, + package : Data::Package?, name : String, version : String, - type : Package::DependencyType? = nil, + type : Data::Package::DependencyType? = nil, *, state : Commands::Install::State, is_direct_dependency : Bool = false, single_resolution : Bool = false, - ancestors : Deque(Package) = Deque(Package).new, + ancestors : Deque(Data::Package) = Deque(Data::Package).new, bust_pinned_cache : Bool = false, - &on_resolve : Package -> _ + &on_resolve : Data::Package -> _ ) Log.debug { "(#{name}@#{version}) Resolving package…" + (type ? " [type: #{type}]" : "") + (package ? " [parent: #{package.key}]" : "") } state.reporter.on_resolving_package @@ -133,6 +134,7 @@ module Zap::Commands::Install::Resolver force_metadata_retrieval = state.install_config.force_metadata_retrieval # Multithreaded dependency resolution (if enabled) state.pipeline.process do + # Infer the parent package parent = package.try { |package| is_direct_dependency ? state.lockfile.get_root(package.name, package.version) : package } # Create the appropriate resolver depending on the version (git, tarball, registry, local folder…) resolver = Resolver.get(state, name, version, parent, type) @@ -228,7 +230,7 @@ module Zap::Commands::Install::Resolver state.reporter.stop package_in_error = "#{name}@#{version}" state.reporter.error(e, package_in_error.colorize.bold.to_s) - exit ErrorCodes::RESOLVER_ERROR.to_i32 + exit Shared::Constants::ErrorCodes::RESOLVER_ERROR.to_i32 end ensure # Report the package as resolved @@ -238,7 +240,7 @@ module Zap::Commands::Install::Resolver # # Private - private def self.flag_transitive_overrides(package : Package, ancestors : Iterable(Package), state : Commands::Install::State) + private def self.flag_transitive_overrides(package : Data::Package, ancestors : Iterable(Data::Package), state : Commands::Install::State) # Check if the package has overrides if (overrides = state.lockfile.overrides) && overrides.size > 0 # A transitive overrides is an 'unsatisfied' override - waiting for an ancestor to match the pattern @@ -250,11 +252,11 @@ module Zap::Commands::Install::Resolver # Concatenate to the transitive overrides {% if flag?(:preview_mt) %} transitive_overrides.try { |to| - (overrides ||= [] of Package::Overrides::Override).concat(to.inner) + (overrides ||= [] of Data::Package::Overrides::Override).concat(to.inner) } {% else %} transitive_overrides.try { |to| - (overrides ||= [] of Package::Overrides::Override).concat(to) + (overrides ||= [] of Data::Package::Overrides::Override).concat(to) } {% end %} overrides.try &.each do |override| @@ -265,7 +267,7 @@ module Zap::Commands::Install::Resolver # Check each ancestor recursively and check if it matches the override pattern ancestors.reverse_each do |ancestor| matches = ancestor.name == parent.name && ( - parent.version == "*" || Utils::Semver.parse(parent.version).satisfies?(ancestor.version) + parent.version == "*" || Semver.parse(parent.version).satisfies?(ancestor.version) ) if matches if parents_index > 0 @@ -279,7 +281,7 @@ module Zap::Commands::Install::Resolver end # Add the override to the ancestor ancestor.transitive_overrides_init { - SafeSet(Package::Overrides::Override).new + Concurrency::SafeSet(Data::Package::Overrides::Override).new } << override end end @@ -287,9 +289,9 @@ module Zap::Commands::Install::Resolver end end - def self.resolve_added_packages(package : Package, *, state : Commands::Install::State, directory : String) + def self.resolve_added_packages(package : Data::Package, *, state : Commands::Install::State, directory : String) # Infer new dependency type based on CLI flags - type = state.install_config.save_dev ? Package::DependencyType::DevDependency : state.install_config.save_optional ? Package::DependencyType::OptionalDependency : Package::DependencyType::Dependency + type = state.install_config.save_dev ? Data::Package::DependencyType::DevDependency : state.install_config.save_optional ? Data::Package::DependencyType::OptionalDependency : Data::Package::DependencyType::Dependency # For each added dependency… pipeline = Pipeline.new state.install_config.added_packages.each do |new_dep| @@ -322,7 +324,7 @@ module Zap::Commands::Install::Resolver raise e end - private def self.apply_package_extensions(metadata : Package, *, state : Commands::Install::State) : Nil + private def self.apply_package_extensions(metadata : Data::Package, *, state : Commands::Install::State) : Nil previous_extensions_shasum = metadata.package_extension_shasum new_extensions_shasum = nil @@ -330,8 +332,8 @@ module Zap::Commands::Install::Resolver if package_extensions = state.context.main_package.zap_config.try(&.package_extensions) # Find matching extensions extensions = package_extensions.select { |extension| - name, version = Utils::Various.parse_key(extension) - name == metadata.name && (!version || Utils::Semver.parse(version).satisfies?(metadata.version)) + name, version = Utils::Misc.parse_key(extension) + name == metadata.name && (!version || Semver.parse(version).satisfies?(metadata.version)) } new_extensions_shasum = extensions.size > 0 ? Digest::MD5.hexdigest(extensions.to_json) : nil diff --git a/packages/commands/install/state.cr b/packages/commands/install/state.cr new file mode 100644 index 0000000..608b25b --- /dev/null +++ b/packages/commands/install/state.cr @@ -0,0 +1,23 @@ +require "data/package" +require "data/lockfile" +require "store" +require "data/npmrc" +require "concurrency/pipeline" +require "core/config" +require "./config" +require "./registry_clients" +require "reporter/interactive" + +module Commands::Install + record State, + config : Core::Config, + install_config : Install::Config, + store : ::Store, + main_package : Data::Package, + lockfile : Data::Lockfile, + context : Core::Config::InferredContext, + npmrc : Data::Npmrc, + registry_clients : RegistryClients, + pipeline : Concurrency::Pipeline = Concurrency::Pipeline.new, + reporter : Reporter = Reporter::Interactive.new +end diff --git a/packages/commands/rebuild/cli.cr b/packages/commands/rebuild/cli.cr new file mode 100644 index 0000000..8527b12 --- /dev/null +++ b/packages/commands/rebuild/cli.cr @@ -0,0 +1,17 @@ +require "../cli" +require "../helpers" +require "./config" + +class Commands::Rebuild::CLI < Commands::CLI + def register(parser : OptionParser, command_config : Core::CommandConfigRef) : Nil + Helpers.command(["rebuild", "rb"], "Rebuild native dependencies.", " [options are passed through]") do + command_config.ref = Rebuild::Config.new(ENV, "ZAP_REBUILD") + + parser.stop + end + end + + private macro rebuild_config + command_config.ref.as(Rebuild::Config) + end +end diff --git a/src/commands/rebuild/config.cr b/packages/commands/rebuild/config.cr similarity index 74% rename from src/commands/rebuild/config.cr rename to packages/commands/rebuild/config.cr index 8ca5fd6..5b746cd 100644 --- a/src/commands/rebuild/config.cr +++ b/packages/commands/rebuild/config.cr @@ -1,7 +1,7 @@ -require "../config" -require "../../utils/macros" +require "utils/macros" +require "core/command_config" -struct Zap::Commands::Rebuild::Config < Zap::Commands::Config +struct Commands::Rebuild::Config < Core::CommandConfig Utils::Macros.record_utils getter packages : Array(String)? = nil diff --git a/src/commands/rebuild/rebuild.cr b/packages/commands/rebuild/rebuild.cr similarity index 77% rename from src/commands/rebuild/rebuild.cr rename to packages/commands/rebuild/rebuild.cr index 1d7856e..6b01935 100644 --- a/src/commands/rebuild/rebuild.cr +++ b/packages/commands/rebuild/rebuild.cr @@ -1,6 +1,6 @@ -module Zap::Commands::Rebuild +module Commands::Rebuild def self.run( - config : Zap::Config, + config : Core::Config, rebuild_config : Rebuild::Config ) rebuild_config = rebuild_config.from_args(ARGV) @@ -17,10 +17,10 @@ module Zap::Commands::Rebuild #{"scope".colorize.blue}: #{context.command_scope.size} package(s) • #{scope_names.sort.join(", ")} TERM end - print NEW_LINE + print Shared::Constants::NEW_LINE end - scripts = [] of Utils::Scripts::ScriptData + scripts = [] of Data::Package::Scripts::ScriptData scope.each do |workspace_or_main_package| if workspace_or_main_package.is_a?(Workspaces::Workspace) @@ -31,26 +31,26 @@ module Zap::Commands::Rebuild end root_path = workspace.try(&.path./ "node_modules") || config.node_modules self.crawl_native_packages(root_path.to_s) do |module_path| - pkg = Package.init?(Path.new(module_path)) + pkg = Data::Package.init?(Path.new(module_path)) next unless pkg filters = rebuild_config.packages if filters && filters.size > 0 matches = filters.any? do |filter| - name, semver = Utils::Various.parse_key(filter) - pkg.name == name && (!semver || Utils::Semver.parse(semver).satisfies?(pkg.version)) + name, semver = Utils::Misc.parse_key(filter) + pkg.name == name && (!semver || Semver.parse(semver).satisfies?(pkg.version)) end next unless matches end - pkg_scripts = pkg.scripts || Zap::Package::LifecycleScripts.new + pkg_scripts = pkg.scripts || Data::Package::LifecycleScripts.new install_args = rebuild_config.flags.try { |flags| " #{flags.join(" ")}" } || "" preinstall_script = pkg_scripts.preinstall.try do |preinstall_script| - Utils::Scripts::ScriptDataNested.new(pkg, module_path, :preinstall, preinstall_script) + Data::Package::Scripts::ScriptDataNested.new(pkg, module_path, :preinstall, preinstall_script) end postinstall_script = pkg_scripts.postinstall.try do |postinstall_script| - Utils::Scripts::ScriptDataNested.new(pkg, module_path, :preinstall, postinstall_script) + Data::Package::Scripts::ScriptDataNested.new(pkg, module_path, :preinstall, postinstall_script) end - script_data = Utils::Scripts::ScriptData.new( + script_data = Data::Package::Scripts::ScriptData.new( pkg, module_path, "install", @@ -62,7 +62,7 @@ module Zap::Commands::Rebuild end end - Utils::Scripts.parallel_run( + Data::Package::Scripts.parallel_run( config: config, scripts: scripts, print_header: false, diff --git a/packages/commands/run/cli.cr b/packages/commands/run/cli.cr new file mode 100644 index 0000000..db4216e --- /dev/null +++ b/packages/commands/run/cli.cr @@ -0,0 +1,39 @@ +require "../cli" +require "../helpers" +require "./config" + +class Commands::Run::CLI < Commands::CLI + def register(parser : OptionParser, command_config : Core::CommandConfigRef) : Nil + Helpers.command(["run", "r"], "Run a package's \"script\" command.", "[options]