From ad41d1e39b10c56d819ef77c4f5eae6c31537fab Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 24 May 2021 12:28:17 +0200
Subject: [PATCH 01/13] Use streamLayeredImage for base image

---
 src/nix/modules/composition/arion-base-image.nix | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/nix/modules/composition/arion-base-image.nix b/src/nix/modules/composition/arion-base-image.nix
index 890c60b..b471313 100644
--- a/src/nix/modules/composition/arion-base-image.nix
+++ b/src/nix/modules/composition/arion-base-image.nix
@@ -6,11 +6,9 @@
 { config, lib, pkgs, ... }: 
 
 let
-
-  tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath));
   name = "arion-base";
 
-  builtImage = pkgs.dockerTools.buildImage {
+  imageExe = pkgs.dockerTools.streamLayeredImage {
     inherit name;
     contents = pkgs.runCommand "minimal-contents" {} ''
         mkdir -p $out/bin $out/usr/bin
@@ -19,6 +17,7 @@ let
       '';
     config = {};
   };
+  inherit (imageExe) imageTag;
 
 in
 
@@ -33,9 +32,9 @@ in
   };
 
   config = {
-    arionBaseImage = "${name}:${tag}";
+    arionBaseImage = "${name}:${imageTag}";
     build.imagesToLoad = lib.mkIf (lib.any (s: s.service.useHostStore) (lib.attrValues config.services)) [
-      { image = builtImage; imageName = name; imageTag = tag; }
+      { imageExe = imageExe; imageName = name; inherit imageTag; }
     ];  
   };
 }
\ No newline at end of file

From 48d3d4b0d79cef96aaff98b15878be4330bca150 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 25 May 2021 19:25:15 +0200
Subject: [PATCH 02/13] Use image.includeStorePaths = false, no more "arion
 base image"

The arion base image was a poor substitute for the customization
layer that only worked for some images.

By modifying dockerTools to export only the customization layer,
we can support arbitrary root contents.
---
 src/nix/modules.nix                           |  1 -
 .../modules/composition/arion-base-image.nix  | 40 ----------------
 src/nix/modules/service/host-store.nix        |  4 +-
 src/nix/modules/service/image.nix             | 48 +++++++++++++++++--
 4 files changed, 45 insertions(+), 48 deletions(-)
 delete mode 100644 src/nix/modules/composition/arion-base-image.nix

diff --git a/src/nix/modules.nix b/src/nix/modules.nix
index b176b2e..ecc73c1 100644
--- a/src/nix/modules.nix
+++ b/src/nix/modules.nix
@@ -3,6 +3,5 @@
     ./modules/composition/host-environment.nix
     ./modules/composition/images.nix
     ./modules/composition/service-info.nix
-    ./modules/composition/arion-base-image.nix
     ./modules/composition/composition.nix
 ]
\ No newline at end of file
diff --git a/src/nix/modules/composition/arion-base-image.nix b/src/nix/modules/composition/arion-base-image.nix
deleted file mode 100644
index b471313..0000000
--- a/src/nix/modules/composition/arion-base-image.nix
+++ /dev/null
@@ -1,40 +0,0 @@
-
-
-# This module is subject to change.
-# In particular, arion-base should use a generic non-service image building system
-
-{ config, lib, pkgs, ... }: 
-
-let
-  name = "arion-base";
-
-  imageExe = pkgs.dockerTools.streamLayeredImage {
-    inherit name;
-    contents = pkgs.runCommand "minimal-contents" {} ''
-        mkdir -p $out/bin $out/usr/bin
-        ln -s /run/system/bin/sh $out/bin/sh
-        ln -s /run/system/usr/bin/env $out/usr/bin/env
-      '';
-    config = {};
-  };
-  inherit (imageExe) imageTag;
-
-in
-
-{
-
-  options = {
-    arionBaseImage = lib.mkOption {
-      type = lib.types.str;
-      description = "Image to use when using useHostStore. Don't use this option yourself. It's going away.";
-      internal = true;
-    };
-  };
-
-  config = {
-    arionBaseImage = "${name}:${imageTag}";
-    build.imagesToLoad = lib.mkIf (lib.any (s: s.service.useHostStore) (lib.attrValues config.services)) [
-      { imageExe = imageExe; imageName = name; inherit imageTag; }
-    ];  
-  };
-}
\ No newline at end of file
diff --git a/src/nix/modules/service/host-store.nix b/src/nix/modules/service/host-store.nix
index 3167608..f50c509 100644
--- a/src/nix/modules/service/host-store.nix
+++ b/src/nix/modules/service/host-store.nix
@@ -29,12 +29,10 @@ in
     };
   };
   config = mkIf config.service.useHostStore {
-    image.nixBuild = false; # no need to build and load
-    service.image = config.composition.arionBaseImage;
+    image.includeStorePaths = false;
     service.environment.NIX_REMOTE = lib.optionalString config.service.useHostNixDaemon "daemon";
     service.volumes = [
       "${config.host.nixStorePrefix}/nix/store:/nix/store${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"
-      "${config.host.nixStorePrefix}${pkgs.buildEnv { name = "container-system-env"; paths = [ pkgs.bashInteractive pkgs.coreutils ]; }}:/run/system${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"
       ] ++ lib.optional config.service.useHostNixDaemon "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket";
     service.command = lib.mkDefault (map escape (config.image.rawConfig.Cmd or []));
   };
diff --git a/src/nix/modules/service/image.nix b/src/nix/modules/service/image.nix
index df78ac2..8d8884c 100644
--- a/src/nix/modules/service/image.nix
+++ b/src/nix/modules/service/image.nix
@@ -1,6 +1,15 @@
 { pkgs, lib, config, options, ... }:
 let
-  inherit (lib) types mkOption;
+  inherit (lib)
+    functionArgs
+    mkOption
+    optionalAttrs
+    types
+    warn
+    ;
+  inherit (pkgs)
+    dockerTools
+    ;
   inherit (types) attrsOf listOf nullOr package str unspecified bool;
 
   # TODO: dummy-config is a useless layer. Nix 2.3 will let us inspect
@@ -9,15 +18,37 @@ let
     (pkgs.writeText "dummy-config.json" (builtins.toJSON config.image.rawConfig))
   ];
 
+  includeStorePathsWarningAndDefault = lib.warn ''
+    You're using a version of Nixpkgs that doesn't support the includeStorePaths
+    parameter in dockerTools.streamLayeredImage. Without this, Arion's
+    useHostStore does not achieve the intended speedup.
+  '' {};
+
   buildOrStreamLayeredImage = args:
-    if pkgs.dockerTools?streamLayeredImage
-    then pkgs.dockerTools.streamLayeredImage args // { isExe = true; }
-    else pkgs.dockerTools.buildLayeredImage args;
+    let
+      args_base = builtins.intersectAttrs
+        {
+          name = null; tag = null; contents = null; config = null;
+          created = null; extraCommands = null; maxLayers = null;
+        }
+        args;
+      acceptedArgs = functionArgs dockerTools.streamLayeredImage;
+      args_no_store = lib.optionalAttrs (!(args.includeStorePaths or true)) (
+        if acceptedArgs ? includeStorePaths
+        then { inherit (args) includeStorePaths; }
+        else includeStorePathsWarningAndDefault
+      );
+      args_streamLayered = args_base // args_no_store;
+    in
+      if dockerTools?streamLayeredImage
+      then dockerTools.streamLayeredImage args_streamLayered // { isExe = true; }
+      else dockerTools.buildLayeredImage args_base;
 
   builtImage = buildOrStreamLayeredImage {
     inherit (config.image)
       name
       contents
+      includeStorePaths
       ;
     config = config.image.rawConfig;
     maxLayers = 100;
@@ -89,6 +120,15 @@ in
          Top level paths in the container.
       '';
     };
+    image.includeStorePaths = mkOption {
+      type = bool;
+      default = true;
+      internal = true;
+      description = ''
+        Include all referenced store paths. You generally want this in your
+        image, unless you load store paths via some other means, like useHostStore = true;
+      '';
+    };
     image.rawConfig = mkOption {
       type = attrsOf unspecified;
       default = {};

From e0e7531f7dee948eefb7b8af925c6cd26411c70a Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 25 May 2021 19:28:08 +0200
Subject: [PATCH 03/13] Add image.enableRecommendedContents for /bin/sh and
 such

---
 docs/modules/ROOT/partials/NixOSOptions.adoc  | 19 ++++++++++
 examples/minimal/arion-compose.nix            |  1 +
 src/nix/modules/service/all-modules.nix       |  1 +
 src/nix/modules/service/image-recommended.nix | 36 +++++++++++++++++++
 4 files changed, 57 insertions(+)
 create mode 100644 src/nix/modules/service/image-recommended.nix

diff --git a/docs/modules/ROOT/partials/NixOSOptions.adoc b/docs/modules/ROOT/partials/NixOSOptions.adoc
index 7709010..63f83d6 100644
--- a/docs/modules/ROOT/partials/NixOSOptions.adoc
+++ b/docs/modules/ROOT/partials/NixOSOptions.adoc
@@ -197,6 +197,25 @@ Default::
 ----
 
 
+No Example:: {blank}
+
+== services.<name>.image.enableRecommendedContents
+
+Add the `/bin/sh` and `/usr/bin/env` symlinks and some lightweight
+files.
+
+
+[discrete]
+=== details
+
+Type:: boolean
+Default::
++
+----
+false
+----
+
+
 No Example:: {blank}
 
 == services.<name>.image.name
diff --git a/examples/minimal/arion-compose.nix b/examples/minimal/arion-compose.nix
index 9531a9f..54a593a 100644
--- a/examples/minimal/arion-compose.nix
+++ b/examples/minimal/arion-compose.nix
@@ -4,6 +4,7 @@
   config.services = {
 
     webserver = {
+      image.enableRecommendedContents = true;
       service.useHostStore = true;
       service.command = [ "sh" "-c" ''
                   cd "$$WEB_ROOT"
diff --git a/src/nix/modules/service/all-modules.nix b/src/nix/modules/service/all-modules.nix
index 18c0432..085f1aa 100644
--- a/src/nix/modules/service/all-modules.nix
+++ b/src/nix/modules/service/all-modules.nix
@@ -5,6 +5,7 @@
   ./host-store.nix
   ./context.nix
   ./image.nix
+  ./image-recommended.nix
   ./nixos.nix
   ./nixos-init.nix
   ../lib/assert.nix
diff --git a/src/nix/modules/service/image-recommended.nix b/src/nix/modules/service/image-recommended.nix
new file mode 100644
index 0000000..a4c5eac
--- /dev/null
+++ b/src/nix/modules/service/image-recommended.nix
@@ -0,0 +1,36 @@
+{ config, lib, pkgs, ... }:
+let
+  inherit (lib)
+    mkIf
+    mkOption
+    types
+    ;
+  inherit (types)
+    bool
+    ;
+
+  recommendedContents = { runCommand, bash, coreutils }:
+    runCommand "recommended-contents" {} ''
+      mkdir -p $out/bin $out/usr/bin $out/var/empty
+      ln -s ${bash}/bin/sh $out/bin/sh
+      ln -s ${coreutils}/bin/env $out/usr/bin/env
+    '';
+in
+{
+  options = {
+    image.enableRecommendedContents = mkOption {
+      type = bool;
+      default = false;
+      description = ''
+        Add the `/bin/sh` and `/usr/bin/env` symlinks and some lightweight
+        files.
+      '';
+    };
+  };
+
+  config = {
+    image.contents = mkIf config.image.enableRecommendedContents [
+      (pkgs.callPackage recommendedContents {})
+    ];
+  };
+}

From 286d56a83cc7495701ed911a5dc7d572bb8900b1 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 26 May 2021 16:57:56 +0200
Subject: [PATCH 04/13] Add test with podman docker socket backend

---
 nix/ci.nix                                    |  9 ++++++--
 nix/default.nix                               | 10 +++++++--
 nix/sources.json                              | 10 ++++-----
 .../testdata/Arion/NixSpec/arion-compose.json | 11 +++++-----
 src/nix/modules/service/nixos-init.nix        |  8 ++++++-
 tests/arion-test/default.nix                  | 21 ++++++++++++++++---
 tests/default.nix                             |  6 +++++-
 7 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/nix/ci.nix b/nix/ci.nix
index 4726fd9..f44c5e2 100644
--- a/nix/ci.nix
+++ b/nix/ci.nix
@@ -13,18 +13,23 @@ dimension "Nixpkgs version" {
       nixpkgsSource = "nixos-19.03";
       enableDoc = false;
       nixosTestIsPerl = true;
+      dockerSupportsSystemd = true;
+      nixosHasPodmanDockerSocket = false;
     };
     "nixos-20_09" = {
       nixpkgsSource = "nixos-20.09";
       isReferenceNixpkgs = true;
       enableDoc = true;
+      dockerSupportsSystemd = true;
+      nixosHasPodmanDockerSocket = false;
     };
     "nixos-unstable" = {
       nixpkgsSource = "nixos-unstable";
       enableDoc = true;
     };
   } (
-    _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true, nixosTestIsPerl ? false }:
+    _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true,
+              nixosTestIsPerl ? false, dockerSupportsSystemd ? false, nixosHasPodmanDockerSocket ? true }:
 
 
       dimension "System" {
@@ -34,7 +39,7 @@ dimension "Nixpkgs version" {
         system: { isReferenceTarget ? false, enableNixOSTests ? true }:
           let
             pkgs = import ./. {
-              inherit system nixosTestIsPerl;
+              inherit system nixosTestIsPerl dockerSupportsSystemd nixosHasPodmanDockerSocket;
               nixpkgsSrc = sources.${nixpkgsSource};
             };
           in
diff --git a/nix/default.nix b/nix/default.nix
index a824a53..a880f2c 100644
--- a/nix/default.nix
+++ b/nix/default.nix
@@ -3,6 +3,8 @@
 , nixpkgsSrc ? sources.${nixpkgsName}
 , system ? builtins.currentSystem
 , nixosTestIsPerl ? false
+, dockerSupportsSystemd ? false
+, nixosHasPodmanDockerSocket ? true
 , ...
 }:
 
@@ -11,8 +13,12 @@ import nixpkgsSrc ({
   config = {
   };
   overlays = [
-    # all the packages are defined there:
-    (_: _: { inherit nixosTestIsPerl; })
+    (_: _: {
+      inherit nixosTestIsPerl;
+      arionTestingFlags = {
+        inherit nixosTestIsPerl dockerSupportsSystemd nixosHasPodmanDockerSocket;
+      };
+    })
     (import ./overlay.nix)
   ];
   inherit system;
diff --git a/nix/sources.json b/nix/sources.json
index 47e0e6c..d68b5da 100644
--- a/nix/sources.json
+++ b/nix/sources.json
@@ -36,15 +36,15 @@
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-unstable": {
-        "branch": "nixos-unstable",
+        "branch": "wip-arion-prs",
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
         "homepage": "https://github.com/NixOS/nixpkgs",
-        "owner": "NixOS",
+        "owner": "hercules-ci",
         "repo": "nixpkgs",
-        "rev": "296793637b22bdb4d23b479879eba0a71c132a66",
-        "sha256": "0j09yih9693w5vjx64ikfxyja1ha7pisygrwrpg3wfz3sssglg69",
+        "rev": "4f5e5257f4385bb178f2b14393c06bada5fdb65e",
+        "sha256": "0m60fvajhx90hamn6arl3ifvbzfwrl2c80v5j44lwcl6n2bjgcmb",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/296793637b22bdb4d23b479879eba0a71c132a66.tar.gz",
+        "url": "https://github.com/hercules-ci/nixpkgs/archive/4f5e5257f4385bb178f2b14393c06bada5fdb65e.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
         "version": ""
     },
diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.json b/src/haskell/testdata/Arion/NixSpec/arion-compose.json
index 479fe96..83d62fc 100644
--- a/src/haskell/testdata/Arion/NixSpec/arion-compose.json
+++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.json
@@ -2,14 +2,14 @@
     "services": {
         "webserver": {
             "command": [
-                "/nix/store/b9w61w4g8sqgrm3rid6ca22krslqghb3-nixos-system-unnamed-19.03.173100.e726e8291b2/init"
+                "/usr/sbin/init"
             ],
             "environment": {
                 "NIX_REMOTE": "",
                 "PATH": "/usr/bin:/run/current-system/sw/bin/",
                 "container": "docker"
             },
-            "image": "arion-base:<HASH>",
+            "image": "webserver:<HASH>",
             "ports": [
                 "8000:80"
             ],
@@ -23,8 +23,7 @@
             "tty": true,
             "volumes": [
                 "/sys/fs/cgroup:/sys/fs/cgroup:ro",
-                "/nix/store:/nix/store:ro",
-                "/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system:ro"
+                "/nix/store:/nix/store:ro"
             ]
         }
     },
@@ -32,8 +31,8 @@
     "x-arion": {
         "images": [
             {
-                "image": "<STOREPATH>",
-                "imageName": "arion-base",
+                "imageExe": "<STOREPATH>",
+                "imageName": "webserver",
                 "imageTag": "<HASH>"
             }
         ],
diff --git a/src/nix/modules/service/nixos-init.nix b/src/nix/modules/service/nixos-init.nix
index d152f0c..38a23a7 100644
--- a/src/nix/modules/service/nixos-init.nix
+++ b/src/nix/modules/service/nixos-init.nix
@@ -24,7 +24,13 @@ in
       ../nixos/default-shell.nix
       (pkgs.path + "/nixos/modules/profiles/minimal.nix")
     ];
-    image.command = [ "${config.nixos.build.toplevel}/init" ];
+    image.command = [ "/usr/sbin/init" ];
+    image.contents = [
+      (pkgs.runCommand "root-init" {} ''
+        mkdir -p $out/usr/sbin
+        ln -s ${config.nixos.build.toplevel}/init $out/usr/sbin/init
+      '')
+    ];
     service.environment.container = "docker";
     service.environment.PATH = "/usr/bin:/run/current-system/sw/bin/";
     service.volumes = [
diff --git a/tests/arion-test/default.nix b/tests/arion-test/default.nix
index dcbd376..e34ff93 100644
--- a/tests/arion-test/default.nix
+++ b/tests/arion-test/default.nix
@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ usePodman ? false, pkgs, lib, ... }:
 
 let
   # To make some prebuilt derivations available in the vm
@@ -6,14 +6,26 @@ let
     inherit modules;
     inherit pkgs;
   };
+
+  inherit (lib)
+    optionalAttrs
+    optionalString
+    ;
+
+  haveSystemd = usePodman || pkgs.arionTestingFlags.dockerSupportsSystemd;
+
 in
 {
   name = "arion-test";
   machine = { pkgs, lib, ... }: {
     environment.systemPackages = [
       pkgs.arion
-    ];
-    virtualisation.docker.enable = true;
+    ] ++ lib.optional usePodman pkgs.docker;
+    virtualisation.docker.enable = !usePodman;
+    virtualisation.podman = optionalAttrs usePodman {
+      enable = true;
+      dockerSocket.enable = true;
+    };
     
     # no caches, because no internet
     nix.binaryCaches = lib.mkForce [];
@@ -38,6 +50,7 @@ in
     ];
 
     virtualisation.memorySize = 1024;
+    virtualisation.diskSize = 8000;
   };
   testScript = ''
     machine.fail("curl --fail localhost:8000")
@@ -71,6 +84,7 @@ in
     #     machine.succeed("cd work && NIX_PATH= arion down")
     #     machine.wait_until_fails("curl --fail localhost:8000")
 
+    ${optionalString haveSystemd ''
     # Tests
     #  - arion exec
     #  - examples/full-nixos
@@ -95,6 +109,7 @@ in
             "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
         )
         machine.wait_until_fails("curl --fail localhost:8000")
+    ''}
 
     # Tests
     #  - examples/nixos-unit
diff --git a/tests/default.nix b/tests/default.nix
index cc9ad49..e134b37 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -1,4 +1,4 @@
-{ pkgs ? import ../pkgs.nix, nixosTestIsPerl ? false }:
+{ pkgs ? import ../pkgs.nix, nixosTestIsPerl ? false, arionTestingFlags ? {} }:
 let
   inherit (pkgs) nixosTest recurseIntoAttrs arion;
 in
@@ -6,6 +6,10 @@ in
 recurseIntoAttrs {
 
   test = if nixosTestIsPerl then nixosTest ./arion-test-perl else nixosTest ./arion-test;
+  testWithPodman =
+    if arionTestingFlags.nixosHasPodmanDockerSocket
+    then nixosTest (pkgs.callPackage ./arion-test { usePodman = true; })
+    else {};
 
   testBuild = arion.build {
 

From 1a24fe963901ad6d477cadb7fca5a88af1fec919 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 26 May 2021 16:59:50 +0200
Subject: [PATCH 05/13] Warn when DynamicUser is used without SYS_ADMIN

---
 src/nix/modules/service/all-modules.nix     |  1 +
 src/nix/modules/service/check-sys_admin.nix | 30 +++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 src/nix/modules/service/check-sys_admin.nix

diff --git a/src/nix/modules/service/all-modules.nix b/src/nix/modules/service/all-modules.nix
index 085f1aa..0b56623 100644
--- a/src/nix/modules/service/all-modules.nix
+++ b/src/nix/modules/service/all-modules.nix
@@ -9,4 +9,5 @@
   ./nixos.nix
   ./nixos-init.nix
   ../lib/assert.nix
+  ./check-sys_admin.nix
 ]
diff --git a/src/nix/modules/service/check-sys_admin.nix b/src/nix/modules/service/check-sys_admin.nix
new file mode 100644
index 0000000..0ea2ee0
--- /dev/null
+++ b/src/nix/modules/service/check-sys_admin.nix
@@ -0,0 +1,30 @@
+{ config, lib, name, ... }:
+let
+  inherit (lib)
+    concatStringsSep
+    optional
+    ;
+
+  dynamicUserServices = lib.attrNames (
+    lib.filterAttrs
+      (k: v:
+        v.enable &&
+        v.serviceConfig.DynamicUser or false)
+      config.nixos.evaluatedConfig.systemd.services
+    );
+
+  
+in
+{
+  config = {
+    warnings = 
+      optional (config.nixos.useSystemd && !(config.service.capabilities.SYS_ADMIN or false) && dynamicUserServices != []) (
+        ''In service ${name}, the following units require `SYS_ADMIN` capability
+          because of DynamicUser.
+          ${concatStringsSep "\n" (map (srv: "    - services.${name}.nixos.configuration.systemd.services.${srv}") dynamicUserServices)}
+          You can avoid DynamicUser or use
+              services.${name}.service.capabilities.SYS_ADMIN = true;
+        ''
+      );
+  };
+}
\ No newline at end of file

From a2f5c9415c193d201775aa2b613ab8705150a982 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 26 May 2021 17:48:15 +0200
Subject: [PATCH 06/13] Update README for 21.05

---
 docs/modules/ROOT/pages/index.adoc | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc
index 37eb06f..3ef02e0 100644
--- a/docs/modules/ROOT/pages/index.adoc
+++ b/docs/modules/ROOT/pages/index.adoc
@@ -39,6 +39,12 @@ Arion allows to compose containers with different granularity:
   * <<NixOS: run full OS>>
   * <<Docker image from DockerHub>>
 
+Full NixOS is supported on
+ * docker-compose + podman with docker socket (NixOS >= 21.05)
+ * docker-compose + docker, before cgroupsv2 (NixOS < 21.05)
+
+`podman-compose` support is currently WIP on a separate branch.
+
 == Installation
 
 === Nix
@@ -52,10 +58,17 @@ $ nix-env -iA arion -f https://github.com/hercules-ci/arion/tarball/master
 Add this module to your NixOS configuration:
 
 ```nix
-{ ... }: {
-  environment.systemPackages = [ (import (builtins.fetchTarball https://github.com/hercules-ci/arion/tarball/master) {}).arion ];
-  virtualisation.docker.enable = true;
-  users.extraUsers.myuser.extraGroups = ["docker"];
+{ pkgs, ... }: {
+  environment.systemPackages = [
+    pkgs.arion
+    pkgs.docker # docker CLI will use podman socket
+  ];
+  virtualisation.docker.enable = false;
+  virtualisation.podman.enable = true;
+  virtualisation.podman.dockerSocket.enable = true;
+
+  # Use your username instead of `myuser`
+  users.extraUsers.myuser.extraGroups = ["podman"];
 }
 ```
 

From 144864d61cf0739f6d79d80e48e7d6c91467f6f4 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 26 May 2021 20:58:59 +0200
Subject: [PATCH 07/13] Drop NixOS 19.03

---
 CHANGELOG.md                      |  2 +-
 nix/ci.nix                        | 15 ++------
 nix/default.nix                   |  4 +--
 nix/sources.json                  | 12 -------
 src/nix/modules/lib/assert.nix    | 13 +++----
 tests/arion-test-perl/README.md   |  4 ---
 tests/arion-test-perl/default.nix | 60 -------------------------------
 tests/default.nix                 |  5 +--
 8 files changed, 14 insertions(+), 101 deletions(-)
 delete mode 100644 tests/arion-test-perl/README.md
 delete mode 100644 tests/arion-test-perl/default.nix

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5a49809..0ee237f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@
 
 * Add a tested example for Traefik with label-based routing.
 
-* Drop obsolete NixOS 19.09 and 20.03 support. It may still be usable there.
+* Drop obsolete NixOS 19.03, 19.09 and 20.03 from CI.
 
 ## 0.1.2.0 -- 2020-03-05
 
diff --git a/nix/ci.nix b/nix/ci.nix
index f44c5e2..a857704 100644
--- a/nix/ci.nix
+++ b/nix/ci.nix
@@ -5,17 +5,6 @@ let
 in
 
 dimension "Nixpkgs version" {
-    "nixos-19_03" = {
-      # flyingcircus.io latest long-term support is based off 19.03
-      # https://flyingcircus.io/doc/
-      # It is nice to have some level of support for their platform,
-      # but we don't guarantee any support.
-      nixpkgsSource = "nixos-19.03";
-      enableDoc = false;
-      nixosTestIsPerl = true;
-      dockerSupportsSystemd = true;
-      nixosHasPodmanDockerSocket = false;
-    };
     "nixos-20_09" = {
       nixpkgsSource = "nixos-20.09";
       isReferenceNixpkgs = true;
@@ -29,7 +18,7 @@ dimension "Nixpkgs version" {
     };
   } (
     _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true,
-              nixosTestIsPerl ? false, dockerSupportsSystemd ? false, nixosHasPodmanDockerSocket ? true }:
+             dockerSupportsSystemd ? false, nixosHasPodmanDockerSocket ? true }:
 
 
       dimension "System" {
@@ -39,7 +28,7 @@ dimension "Nixpkgs version" {
         system: { isReferenceTarget ? false, enableNixOSTests ? true }:
           let
             pkgs = import ./. {
-              inherit system nixosTestIsPerl dockerSupportsSystemd nixosHasPodmanDockerSocket;
+              inherit system dockerSupportsSystemd nixosHasPodmanDockerSocket;
               nixpkgsSrc = sources.${nixpkgsSource};
             };
           in
diff --git a/nix/default.nix b/nix/default.nix
index a880f2c..1bafe0e 100644
--- a/nix/default.nix
+++ b/nix/default.nix
@@ -2,7 +2,6 @@
 , nixpkgsName ? "nixos-unstable"
 , nixpkgsSrc ? sources.${nixpkgsName}
 , system ? builtins.currentSystem
-, nixosTestIsPerl ? false
 , dockerSupportsSystemd ? false
 , nixosHasPodmanDockerSocket ? true
 , ...
@@ -14,9 +13,8 @@ import nixpkgsSrc ({
   };
   overlays = [
     (_: _: {
-      inherit nixosTestIsPerl;
       arionTestingFlags = {
-        inherit nixosTestIsPerl dockerSupportsSystemd nixosHasPodmanDockerSocket;
+        inherit dockerSupportsSystemd nixosHasPodmanDockerSocket;
       };
     })
     (import ./overlay.nix)
diff --git a/nix/sources.json b/nix/sources.json
index d68b5da..29b95d2 100644
--- a/nix/sources.json
+++ b/nix/sources.json
@@ -11,18 +11,6 @@
         "url": "https://github.com/nmattia/niv/archive/fad2a6cbfb2e7cdebb7cb0ad2f5cc91e2c9bc06b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "nixos-19.03": {
-        "branch": "nixos-19.03",
-        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
-        "homepage": "https://github.com/NixOS/nixpkgs",
-        "owner": "NixOS",
-        "repo": "nixpkgs-channels",
-        "rev": "34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59",
-        "sha256": "11z6ajj108fy2q5g8y4higlcaqncrbjm3dnv17pvif6avagw4mcb",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "nixos-20.09": {
         "branch": "nixos-20.09",
         "description": "Nix Packages collection",
diff --git a/src/nix/modules/lib/assert.nix b/src/nix/modules/lib/assert.nix
index f9bf4a8..c1940d2 100644
--- a/src/nix/modules/lib/assert.nix
+++ b/src/nix/modules/lib/assert.nix
@@ -3,14 +3,15 @@
 # based on nixpkgs/nixos/modules/system/activation/top-level.nix
 
 let
-  inherit (lib) filter concatStringsSep types mkOption;
-
-  # lib.showWarnings since 19.09
-  showWarnings = warnings: res: lib.fold (w: x: lib.warn w x) res warnings;
-  warn = msg: builtins.trace "warning: ${msg}";
+  inherit (lib)
+    concatStringsSep
+    filter
+    mkOption
+    showWarnings
+    types
+    ;
 
   # Handle assertions and warnings
-
   failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions);
 
   assertWarn = if failedAssertions != []
diff --git a/tests/arion-test-perl/README.md b/tests/arion-test-perl/README.md
deleted file mode 100644
index 736afb7..0000000
--- a/tests/arion-test-perl/README.md
+++ /dev/null
@@ -1,4 +0,0 @@
-
-This test suite exists only to keep tests around for older versions of NixOS.
-
-This will be removed when 19.09 becomes irrelevant.
diff --git a/tests/arion-test-perl/default.nix b/tests/arion-test-perl/default.nix
deleted file mode 100644
index 34c42b3..0000000
--- a/tests/arion-test-perl/default.nix
+++ /dev/null
@@ -1,60 +0,0 @@
-{ pkgs, ... }:
-
-let
-  # To make some prebuilt derivations available in the vm
-  preEval = modules: import ../../src/nix/eval-composition.nix {
-    inherit modules;
-    inherit pkgs;
-  };
-in
-{
-  name = "arion-test";
-  machine = { pkgs, lib, ... }: {
-    environment.systemPackages = [
-      pkgs.arion
-    ];
-    virtualisation.docker.enable = true;
-    
-    # no caches, because no internet
-    nix.binaryCaches = lib.mkForce [];
-
-    # FIXME: Sandbox seems broken with current version of NixOS test
-    #        w/ writable store. Error:
-    #        machine# error: linking '/nix/store/7r8z2zvhwda85pgpdn5hzzz6hs1njklc-stdenv-linux.drv.chroot/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh' to '/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh': Operation not permitted
-    #
-    #        There should be no reason why arion can't run without
-    #        sandboxing, so please re-enable.
-    nix.useSandbox = false;
-
-    virtualisation.writableStore = true;
-    virtualisation.pathsInNixDB = [
-      # Pre-build the image because we don't want to build the world
-      # in the vm.
-      (preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml
-      (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml
-      (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml
-      pkgs.stdenv
-    ];
-
-    virtualisation.memorySize = 512;
-  };
-  testScript = ''
-    $machine->fail("curl localhost:8000");
-    $machine->succeed("docker --version");
-
-    my $makeSubtest = sub {
-      my ( $subtestName, $exampleSrc ) = @_;
-
-      subtest $subtestName => sub {
-        $machine->succeed("rm -rf work && cp -frT $exampleSrc work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
-        $machine->waitUntilSucceeds("curl localhost:8000");
-        $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down");
-        $machine->waitUntilFails("curl localhost:8000");
-      };
-    };
-
-    $makeSubtest->("minimal", "${../../examples/minimal}");
-    $makeSubtest->("full-nixos", "${../../examples/full-nixos}");
-    $makeSubtest->("nixos-unit", "${../../examples/nixos-unit}");
-  '';
-}
diff --git a/tests/default.nix b/tests/default.nix
index e134b37..d7a3c67 100644
--- a/tests/default.nix
+++ b/tests/default.nix
@@ -1,11 +1,12 @@
-{ pkgs ? import ../pkgs.nix, nixosTestIsPerl ? false, arionTestingFlags ? {} }:
+{ pkgs ? import ../pkgs.nix, arionTestingFlags ? {} }:
 let
   inherit (pkgs) nixosTest recurseIntoAttrs arion;
 in
 
 recurseIntoAttrs {
 
-  test = if nixosTestIsPerl then nixosTest ./arion-test-perl else nixosTest ./arion-test;
+  test = nixosTest ./arion-test;
+
   testWithPodman =
     if arionTestingFlags.nixosHasPodmanDockerSocket
     then nixosTest (pkgs.callPackage ./arion-test { usePodman = true; })

From 9dd1ab0568596091fac4e202bac313c1fc19b656 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Wed, 26 May 2021 21:09:17 +0200
Subject: [PATCH 08/13] Update change log

---
 CHANGELOG.md | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ee237f..0eb33e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,13 +2,28 @@
 
 ## Next
 
+### Changed
+
+* `useHostStore` now uses an image derived from the `image.*` options. You may
+  need to enable `enableRecommendedContents` because with this change, files
+  like `/bin/sh` aren't added by default anymore.
+
+* Drop obsolete NixOS 19.03, 19.09 and 20.03 from CI.
+
+### Added
+
+* NixOS-based containers can now run on Podman when it is configured to provide a docker socket. See the [installation docs](https://docs.hercules-ci.com/arion/#_nixos).
+
 * Support `service.dns`, for overriding the DNS servers used by containers.
 
 * Support `service.labels`, which is useful for autodiscovery among other things.
 
 * Add a tested example for Traefik with label-based routing.
 
-* Drop obsolete NixOS 19.03, 19.09 and 20.03 from CI.
+* Add a `flake.nix` and an experimental flake example
+
+* Add a warning when systemd `DynamicUser` is used but not available to the
+  container.
 
 ## 0.1.2.0 -- 2020-03-05
 

From dcc5b1e3ceb120c1f75da1d4fec0a6c842bab2fd Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 31 May 2021 15:14:24 +0200
Subject: [PATCH 09/13] Test and fix that we only load images when needed

---
 src/haskell/lib/Arion/Images.hs |  6 +++++-
 tests/arion-test/default.nix    | 38 +++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/src/haskell/lib/Arion/Images.hs b/src/haskell/lib/Arion/Images.hs
index 350ac93..e2899f4 100644
--- a/src/haskell/lib/Arion/Images.hs
+++ b/src/haskell/lib/Arion/Images.hs
@@ -22,7 +22,11 @@ loadImages requestedImages = do
   loaded <- getDockerImages
 
   let
-    isNew i = (imageName i <> ":" <> imageTag i) `notElem` loaded
+    isNew i = 
+      -- On docker, the image name is unmodified
+      (imageName i <> ":" <> imageTag i) `notElem` loaded
+      -- -- On podman, you automatically get a localhost prefix
+        && ("localhost/" <> imageName i <> ":" <> imageTag i) `notElem` loaded
 
   traverse_ loadImage . filter isNew $ requestedImages
 
diff --git a/tests/arion-test/default.nix b/tests/arion-test/default.nix
index e34ff93..3d05648 100644
--- a/tests/arion-test/default.nix
+++ b/tests/arion-test/default.nix
@@ -70,6 +70,44 @@ in
         )
         machine.wait_until_fails("curl --fail localhost:8000")
 
+    # Tests
+    #  - running same image again doesn't require a `docker load`
+    with subtest("docker load only once"):
+        # We assume image loading relies on the `docker images` and `docker load` commands, so this should fail
+        machine.fail(
+            "export REAL_DOCKER=$(which docker); rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' PATH=\"${pkgs.writeScriptBin "docker" ''
+              #!${pkgs.runtimeShell} -eu
+              echo 1>&2 "This failure is expected. Args were" "$@"
+              echo "$@" >/tmp/docker-args
+              exit 1
+            ''}/bin:$PATH\" arion up -d"
+        )
+        machine.succeed(
+            "export REAL_DOCKER=$(which docker); rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' PATH=\"${pkgs.writeScriptBin "docker" ''
+              #!${pkgs.runtimeShell} -eu
+              case $1 in
+                load)
+                  echo 1>&2 "arion must not docker load when upping the same deployment for the second time"
+                  exit 1
+                  ;;
+                images)
+                  echo 1>&2 "execing docker to list images"
+                  exec $REAL_DOCKER "$@"
+                  ;;
+                *)
+                  echo 1>&2 "Unknown docker invocation. This may be a shortcoming of this docker mock."
+                  echo 1>&2 "Invocation: docker" "$@"
+                  ;;
+              esac
+            ''}/bin:$PATH\" arion up -d"
+        )
+        machine.wait_until_succeeds("curl --fail localhost:8000")
+        machine.succeed(
+            "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
+        )
+        machine.wait_until_fails("curl --fail localhost:8000")
+
+
     # Tests
     #  - examples/flake
     # This _test_ doesn't work because flake-compat fetches the github

From e73710caf92dbc9c9f8f6f4e153adbc377a84138 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 31 May 2021 15:15:03 +0200
Subject: [PATCH 10/13] examples/full-nixos: Disable nscd

---
 examples/full-nixos/arion-compose.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/full-nixos/arion-compose.nix b/examples/full-nixos/arion-compose.nix
index 1429355..1524e3a 100644
--- a/examples/full-nixos/arion-compose.nix
+++ b/examples/full-nixos/arion-compose.nix
@@ -4,6 +4,8 @@
     nixos.configuration.boot.tmpOnTmpfs = true;
     nixos.configuration.services.nginx.enable = true;
     nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
+    nixos.configuration.services.nscd.enable = false;
+    nixos.configuration.system.nssModules = lib.mkForce [];
     nixos.configuration.systemd.services.nginx.serviceConfig.AmbientCapabilities = 
       lib.mkForce [ "CAP_NET_BIND_SERVICE" ];
     service.useHostStore = true;

From 2b46a9b5f636711643629f70f1bbd6d43a9560d3 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 31 May 2021 15:17:19 +0200
Subject: [PATCH 11/13] Format

---
 src/haskell/lib/Arion/Images.hs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/haskell/lib/Arion/Images.hs b/src/haskell/lib/Arion/Images.hs
index e2899f4..da2ad3c 100644
--- a/src/haskell/lib/Arion/Images.hs
+++ b/src/haskell/lib/Arion/Images.hs
@@ -1,7 +1,7 @@
 {-# LANGUAGE DeriveGeneric #-}
 {-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE OverloadedStrings #-}
-module Arion.Images 
+module Arion.Images
   ( loadImages
   ) where
 
@@ -22,7 +22,7 @@ loadImages requestedImages = do
   loaded <- getDockerImages
 
   let
-    isNew i = 
+    isNew i =
       -- On docker, the image name is unmodified
       (imageName i <> ":" <> imageTag i) `notElem` loaded
       -- -- On podman, you automatically get a localhost prefix
@@ -31,19 +31,19 @@ loadImages requestedImages = do
   traverse_ loadImage . filter isNew $ requestedImages
 
 loadImage :: Image -> IO ()
-loadImage (Image { image = Just imgPath, imageName = name }) =
+loadImage Image { image = Just imgPath, imageName = name } =
   withFile (toS imgPath) ReadMode $ \fileHandle -> do
   let procSpec = (Process.proc "docker" [ "load" ]) {
           Process.std_in = Process.UseHandle fileHandle
         }
   Process.withCreateProcess procSpec $ \_in _out _err procHandle -> do
-    e <- Process.waitForProcess procHandle 
+    e <- Process.waitForProcess procHandle
     case e of
       ExitSuccess -> pass
       ExitFailure code ->
         panic $ "docker load failed with exit code " <> show code <> " for image " <> name <> " from path " <> imgPath
 
-loadImage (Image { imageExe = Just imgExe, imageName = name }) = do
+loadImage Image { imageExe = Just imgExe, imageName = name } = do
   let loadSpec = (Process.proc "docker" [ "load" ]) { Process.std_in = Process.CreatePipe }
   Process.withCreateProcess loadSpec $ \(Just inHandle) _out _err loadProcHandle -> do
     let streamSpec = Process.proc (toS imgExe) []
@@ -61,11 +61,11 @@ loadImage (Image { imageExe = Just imgExe, imageName = name }) = do
             _ -> pass
           pass
 
-loadImage (Image { imageName = name }) = do
+loadImage Image { imageName = name } = do
   panic $ "image " <> name <> " doesn't specify an image file or imageExe executable"
 
 
 getDockerImages :: IO [TaggedImage]
 getDockerImages = do
   let procSpec = Process.proc "docker" [ "images",  "--filter", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}" ]
-  (map toS . T.lines . toS) <$> Process.readCreateProcess procSpec ""
+  map toS . T.lines . toS <$> Process.readCreateProcess procSpec ""

From 07998216c4d9f12a120c01826adc37370509969c Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 31 May 2021 15:23:54 +0200
Subject: [PATCH 12/13] Update nixos-unstable

---
 nix/sources.json | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/nix/sources.json b/nix/sources.json
index 29b95d2..b98b509 100644
--- a/nix/sources.json
+++ b/nix/sources.json
@@ -24,15 +24,15 @@
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-unstable": {
-        "branch": "wip-arion-prs",
+        "branch": "master",
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
         "homepage": "https://github.com/NixOS/nixpkgs",
-        "owner": "hercules-ci",
+        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4f5e5257f4385bb178f2b14393c06bada5fdb65e",
-        "sha256": "0m60fvajhx90hamn6arl3ifvbzfwrl2c80v5j44lwcl6n2bjgcmb",
+        "rev": "97c3d70a39070547a8342f7ee6f5c4a560282179",
+        "sha256": "1pkagmf42n3v4bjk8jr23hcwpa5qy21w0psi0jbdrbsgpp6rchqa",
         "type": "tarball",
-        "url": "https://github.com/hercules-ci/nixpkgs/archive/4f5e5257f4385bb178f2b14393c06bada5fdb65e.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/97c3d70a39070547a8342f7ee6f5c4a560282179.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
         "version": ""
     },

From a7c7ec3a0388e50495562b556e431627c64e6db8 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Mon, 31 May 2021 15:24:20 +0200
Subject: [PATCH 13/13] Update niv boilerplate

---
 nix/sources.nix | 129 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 83 insertions(+), 46 deletions(-)

diff --git a/nix/sources.nix b/nix/sources.nix
index 8a725cb..b796fff 100644
--- a/nix/sources.nix
+++ b/nix/sources.nix
@@ -6,52 +6,63 @@ let
   # The fetchers. fetch_<type> fetches specs of type <type>.
   #
 
-  fetch_file = pkgs: spec:
-    if spec.builtin or true then
-      builtins_fetchurl { inherit (spec) url sha256; }
-    else
-      pkgs.fetchurl { inherit (spec) url sha256; };
+  fetch_file = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      else
+        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
 
-  fetch_tarball = pkgs: spec:
-    if spec.builtin or true then
-      builtins_fetchTarball { inherit (spec) url sha256; }
-    else
-      pkgs.fetchzip { inherit (spec) url sha256; };
+  fetch_tarball = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      else
+        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
 
-  fetch_git = spec:
-    builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+  fetch_git = name: spec:
+    let
+      ref =
+        if spec ? ref then spec.ref else
+          if spec ? branch then "refs/heads/${spec.branch}" else
+            if spec ? tag then "refs/tags/${spec.tag}" else
+              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+    in
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
 
-  fetch_builtin-tarball = spec:
-    builtins.trace
-      ''
-        WARNING:
-          The niv type "builtin-tarball" will soon be deprecated. You should
-          instead use `builtin = true`.
+  fetch_local = spec: spec.path;
 
-          $ niv modify <package> -a type=tarball -a builtin=true
-      ''
-      builtins_fetchTarball { inherit (spec) url sha256; };
+  fetch_builtin-tarball = name: throw
+    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=tarball -a builtin=true'';
 
-  fetch_builtin-url = spec:
-    builtins.trace
-      ''
-        WARNING:
-          The niv type "builtin-url" will soon be deprecated. You should
-          instead use `builtin = true`.
-
-          $ niv modify <package> -a type=file -a builtin=true
-      ''
-      (builtins_fetchurl { inherit (spec) url sha256; });
+  fetch_builtin-url = name: throw
+    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=file -a builtin=true'';
 
   #
   # Various helpers
   #
 
+  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
+  sanitizeName = name:
+    (
+      concatMapStrings (s: if builtins.isList s then "-" else s)
+        (
+          builtins.split "[^[:alnum:]+._?=-]+"
+            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
+        )
+    );
+
   # The set of packages used when specs are fetched using non-builtins.
-  mkPkgs = sources:
+  mkPkgs = sources: system:
     let
       sourcesNixpkgs =
-        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
       hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
       hasThisAsNixpkgsPath = <nixpkgs> == ./.;
     in
@@ -71,14 +82,24 @@ let
 
     if ! builtins.hasAttr "type" spec then
       abort "ERROR: niv spec ${name} does not have a 'type' attribute"
-    else if spec.type == "file" then fetch_file pkgs spec
-    else if spec.type == "tarball" then fetch_tarball pkgs spec
-    else if spec.type == "git" then fetch_git spec
-    else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
-    else if spec.type == "builtin-url" then fetch_builtin-url spec
+    else if spec.type == "file" then fetch_file pkgs name spec
+    else if spec.type == "tarball" then fetch_tarball pkgs name spec
+    else if spec.type == "git" then fetch_git name spec
+    else if spec.type == "local" then fetch_local spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then fetch_builtin-url name
     else
       abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
 
+  # If the environment variable NIV_OVERRIDE_${name} is set, then use
+  # the path directly as opposed to the fetched source.
+  replace = name: drv:
+    let
+      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
+    in
+      if ersatz == "" then drv else ersatz;
+
   # Ports of functions for older nix versions
 
   # a Nix version of mapAttrs if the built-in doesn't exist
@@ -87,23 +108,37 @@ let
     listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
   );
 
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatMapStrings = f: list: concatStrings (map f list);
+  concatStrings = builtins.concatStringsSep "";
+
+  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
+  optionalAttrs = cond: as: if cond then as else {};
+
   # fetchTarball version that is compatible between all the versions of Nix
-  builtins_fetchTarball = { url, sha256 }@attrs:
+  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchTarball;
     in
       if lessThan nixVersion "1.12" then
-        fetchTarball { inherit url; }
+        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
       else
         fetchTarball attrs;
 
   # fetchurl version that is compatible between all the versions of Nix
-  builtins_fetchurl = { url, sha256 }@attrs:
+  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchurl;
     in
       if lessThan nixVersion "1.12" then
-        fetchurl { inherit url; }
+        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
       else
         fetchurl attrs;
 
@@ -115,14 +150,15 @@ let
         then abort
           "The values in sources.json should not have an 'outPath' attribute"
         else
-          spec // { outPath = fetch config.pkgs name spec; }
+          spec // { outPath = replace name (fetch config.pkgs name spec); }
     ) config.sources;
 
   # The "config" used by the fetchers
   mkConfig =
-    { sourcesFile ? ./sources.json
-    , sources ? builtins.fromJSON (builtins.readFile sourcesFile)
-    , pkgs ? mkPkgs sources
+    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
+    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , system ? builtins.currentSystem
+    , pkgs ? mkPkgs sources system
     }: rec {
       # The sources, i.e. the attribute set of spec name to spec
       inherit sources;
@@ -130,5 +166,6 @@ let
       # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
       inherit pkgs;
     };
+
 in
 mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }