Single package fixpoint step 2: introduce mkPackage

lib/customisation.nix

@@ -301,7 +301,26 @@ rec {
 
     in
     if missingArgs == { } then
-      makeOverridable f allArgs
+      if fargs ? mkPackage then
+        if fargs != { mkPackage = false; } then
+          # If your file is e.g. an attrset with a package and something else,
+          # use `{ callPackage }: { foo = callPackage ({ mkPackage}: ...); ... }`
+          # Explaining that in the error message would be too verbose and confusing.
+          # TODO: link to docs
+          throw ''
+            callPackage: A package function that uses `mkPackage` must only have `{ mkPackage }:` as its arguments.
+            Any other dependencies should be taken from the `mkPackage` callback.
+            Otherwise, such dependencies will not be overridable.
+
+            To illustrate, a dependency `foo` can be retrieved with:
+            { mkPackage }: mkPackage ({ layers, foo }: [
+              ...
+            ])
+          ''
+        else
+          f allArgs
+      else
+        makeOverridable f allArgs
     # This needs to be an abort so it can't be caught with `builtins.tryEval`,
     # which is used by nix-env and ofborg to filter out packages that don't evaluate.
     # This way we're forced to fix such errors in Nixpkgs,
@@ -446,9 +465,12 @@ rec {
 
       commonAttrs =
         {
-          inherit (drv) name system meta;
+          inherit (drv) name meta;
           inherit outputs;
         }
+        // optionalAttrs (drv ? system) {
+          inherit (drv) system;
+        }
         // optionalAttrs (drv._hydraAggregate or false) {
           _hydraAggregate = true;
           constituents = map hydraJob (flatten drv.constituents);

lib/default.nix

@@ -80,7 +80,7 @@ let
       genericClosure readFile;
     inherit (self.fixedPoints) fix fix' converge extends composeExtensions
       composeManyExtensions makeExtensible makeExtensibleWithCustomName
-      toExtension;
+      encapsulate toExtension;
     inherit (self.attrsets) attrByPath hasAttrByPath setAttrByPath
       getAttrFromPath attrVals attrNames attrValues getAttrs catAttrs filterAttrs
       filterAttrsRecursive foldlAttrs foldAttrs collect nameValuePair mapAttrs

lib/fixed-points.nix

@@ -451,6 +451,96 @@ rec {
       }
     );
 
+  /*
+    Creates an overridable attrset with encapsulation.
+
+    This is like `makeExtensible`, but only the `public` attribute of the fixed
+    point is returned.
+
+    Synopsis:
+
+        r = encapsulate (final@{extend, ...}: {
+
+          # ... private attributes for `final` ...
+
+          public = {
+            # ... returned attributes for r, in terms of `final` ...
+            inherit extend; # optional, don't invoke too often; see below
+          };
+        })
+
+        s = r.extend (final: previous: {
+
+          # ... updates to private attributes ...
+
+          # optionally
+          public = previous.public // {
+            # ... updates to public attributes ...
+          };
+        })
+
+    = Performance
+
+    The `extend` function evaluates the whole fixed point all over, reusing
+    no "intermediate results" from the existing object.
+    This is necessary, because `final` has changed.
+    So the cost is quadratic; O(n^2) where n = number of chained invocations.
+    This has consequences for interface design.
+    Although enticing, `extend` is not suitable for directly implementing "fluent interfaces", where the caller makes many calls to `extend` via domain-specific "setters" or `with*` functions.
+    Fluent interfaces can not be implemented efficiently in Nix and have very little to offer over attribute sets in terms of usability.*
+
+    Example:
+
+        # cd nixpkgs; nix repl lib
+
+        nix-repl> multiplier = encapsulate (self: {
+          a = 1;
+          b = 1;
+          public = {
+            r = self.a * self.b;
+
+            # Publishing extend makes the attrset open for any kind of change.
+            inherit (self) extend;
+
+            # Instead, or additionally, you can add domain-specific functions.
+            # Offer a single method with multiple arguments, and not a
+            # "fluent interface" of a method per argument, because all extension
+            # functions are called for every `extend`. See the Performance section.
+            withParams = args@{ a ? null, b ? null }: # NB: defaults are not used
+              self.extend (self: super: args);
+
+          };
+        })
+
+        nix-repl> multiplier
+        { extend = «lambda»; r = 1; withParams =«lambda»; }
+
+        nix-repl> multiplier.withParams { a = 42; b = 10; }
+        { extend = «lambda»; r = 420; withParams =«lambda»; }
+
+        nix-repl> multiplier3 = multiplier.extend (self: super: {
+          c = 1;
+          public = super.public // {
+            r = super.public.r * self.c;
+          };
+        })
+
+        nix-repl> multiplier3.extend (self: super: { a = 2; b = 3; c = 10; })
+        { extend = «lambda»; r = 60; withParams =«lambda»; }
+
+    (*) Final note on Fluent APIs: While the asymptotic complexity can be fixed
+        by avoiding overlay extension or perhaps using it only at the end of the
+        chain only, one problem remains. Every method invocation has to produce
+        a new, immutable state value, which means copying the whole state up to
+        that point.
+  */
+  encapsulate =
+    layerZero:
+    let
+      fixed = layerZero ({ extend = f: encapsulate (extends f layerZero); } // fixed);
+    in
+    fixed.public;
+
   /**
     Convert to an extending function (overlay).
 

pkgs/build-support/docker/examples.nix

@@ -27,6 +27,14 @@ let
   };
   evalMinimalConfig = module: nixosLib.evalModules { modules = [ module ]; };
 
+  # TODO: Base this on `devShell` attribute provided by the package, instead
+  #       of prying the internals.
+  #       Context: https://github.com/NixOS/nix/issues/7501
+  getGuts = drvPkg: drvPkg // {
+    inherit (drvPkg.internals) drvAttrs;
+    outputs = drvPkg.internals.setup.outputs or [ "out" ];
+  };
+
   nginxArguments =
     let
       nginxPort = "80";
@@ -673,7 +681,7 @@ rec {
     fakeRootCommands = ''
       mkdir -p ./home/alice
       chown 1000 ./home/alice
-      ln -s ${
+      ln -s ${getGuts (
         pkgs.hello.overrideAttrs (
           finalAttrs: prevAttrs: {
             # A unique `hello` to make sure that it isn't included via another mechanism by accident.
@@ -687,7 +695,7 @@ rec {
             };
           }
         )
-      } ./hello
+      )} ./hello
     '';
   };
 
@@ -869,7 +877,7 @@ rec {
   nix-shell-basic = streamNixShellImage {
     name = "nix-shell-basic";
     tag = "latest";
-    drv = pkgs.hello;
+    drv = getGuts pkgs.hello;
   };
 
   nix-shell-hook = streamNixShellImage {
@@ -972,7 +980,7 @@ rec {
   nix-shell-build-derivation = streamNixShellImage {
     name = "nix-shell-build-derivation";
     tag = "latest";
-    drv = pkgs.hello;
+    drv = getGuts pkgs.hello;
     run = ''
       buildDerivation
       $out/bin/hello

pkgs/build-support/node/build-npm-package/layer.nix

@@ -0,0 +1,75 @@
+/**
+  `mkPackage` layer for building NPM packages.
+*/
+{
+  lib,
+  stdenv,
+  fetchNpmDeps,
+  buildPackages,
+  nodejs,
+  cctools,
+}:
+
+let
+  # The fetcher needs the unpack related attributes
+  fetchInherited = {
+    src = null;
+    srcs = null;
+    sourceRoot = null;
+    prePatch = null;
+    patches = null;
+    postPatch = null;
+    patchFlags = null;
+  };
+in
+this: old:
+let
+  inherit (this) deps name version;
+in
+{
+  deps = {
+    inherit (deps.nodejs) fetchNpmDeps;
+    npmHooks = buildPackages.npmHooks.override {
+      inherit (deps) nodejs;
+    };
+    inherit (deps.npmHooks) npmConfigHook npmBuildHook npmInstallHook;
+  } // old.deps;
+  /**
+    Arguments for fetchNpmDeps
+  */
+  npmFetch =
+    {
+      name = "${name}-${version}-npm-deps";
+      hash = throw "Please specify npmFetch.hash in the package definition of ${name}";
+      forceEmptyCache = false;
+      forceGitDeps = false;
+      patchFlags = "";
+      postPatch = "";
+      prePatch = "";
+    }
+    // builtins.intersectAttrs fetchInherited this.setup
+    // old.npmFetch;
+  setup = old.setup or { } // {
+    inherit (deps) nodejs;
+    npmDeps = fetchNpmDeps this.npmFetch;
+    npmPruneFlags = old.setup.npmPruneFlags or this.setup.npmInstallFlags or [ ];
+    npmBuildScript = "build";
+    nativeBuildInputs =
+      old.setup.nativeBuildInputs
+      ++ [
+        deps.nodejs
+        deps.npmConfigHook
+        deps.npmBuildHook
+        deps.npmInstallHook
+        deps.nodejs.python
+      ]
+      ++ lib.optionals stdenv.hostPlatform.isDarwin [ cctools ];
+    buildInputs = old.setup.buildInputs or [ ] ++ [ deps.nodejs ];
+    strictDeps = true;
+    # Stripping takes way too long with the amount of files required by a typical Node.js project.
+    dontStrip = old.dontStrip or true;
+  };
+  meta = (old.meta or { }) // {
+    platforms = old.meta.platforms or deps.nodejs.meta.platforms;
+  };
+}