Merge pull request #117 from hercules-ci/nixos-21.05-podman-preparation

NixOS 21.05/podman preparation

CHANGELOG.md

@@ -2,13 +2,28 @@
 
 ## Next
 
+### Changed
+
+* `useHostStore` now uses an image derived from the `image.*` options. You may
+  need to enable `enableRecommendedContents` because with this change, files
+  like `/bin/sh` aren't added by default anymore.
+
+* Drop obsolete NixOS 19.03, 19.09 and 20.03 from CI.
+
+### Added
+
+* NixOS-based containers can now run on Podman when it is configured to provide a docker socket. See the [installation docs](https://docs.hercules-ci.com/arion/#_nixos).
+
 * Support `service.dns`, for overriding the DNS servers used by containers.
 
 * Support `service.labels`, which is useful for autodiscovery among other things.
 
 * Add a tested example for Traefik with label-based routing.
 
-* Drop obsolete NixOS 19.09 and 20.03 support. It may still be usable there.
+* Add a `flake.nix` and an experimental flake example
+
+* Add a warning when systemd `DynamicUser` is used but not available to the
+  container.
 
 ## 0.1.2.0 -- 2020-03-05
 

docs/modules/ROOT/pages/index.adoc

@@ -39,6 +39,12 @@ Arion allows to compose containers with different granularity:
   * <<NixOS: run full OS>>
   * <<Docker image from DockerHub>>
 
+Full NixOS is supported on
+ * docker-compose + podman with docker socket (NixOS >= 21.05)
+ * docker-compose + docker, before cgroupsv2 (NixOS < 21.05)
+
+`podman-compose` support is currently WIP on a separate branch.
+
 == Installation
 
 === Nix
@@ -52,10 +58,17 @@ $ nix-env -iA arion -f https://github.com/hercules-ci/arion/tarball/master
 Add this module to your NixOS configuration:
 
 ```nix
-{ ... }: {
+{ pkgs, ... }: {
-  environment.systemPackages = [ (import (builtins.fetchTarball https://github.com/hercules-ci/arion/tarball/master) {}).arion ];
+  environment.systemPackages = [
-  virtualisation.docker.enable = true;
+    pkgs.arion
-  users.extraUsers.myuser.extraGroups = ["docker"];
+    pkgs.docker # docker CLI will use podman socket
+  ];
+  virtualisation.docker.enable = false;
+  virtualisation.podman.enable = true;
+  virtualisation.podman.dockerSocket.enable = true;
+
+  # Use your username instead of `myuser`
+  users.extraUsers.myuser.extraGroups = ["podman"];
 }
 ```
 

docs/modules/ROOT/partials/NixOSOptions.adoc

@@ -197,6 +197,25 @@ Default::
 ----
 
 
+No Example:: {blank}
+
+== services.<name>.image.enableRecommendedContents
+
+Add the `/bin/sh` and `/usr/bin/env` symlinks and some lightweight
+files.
+
+
+[discrete]
+=== details
+
+Type:: boolean
+Default::
++
+----
+false
+----
+
+
 No Example:: {blank}
 
 == services.<name>.image.name

examples/full-nixos/arion-compose.nix

@@ -4,6 +4,8 @@
     nixos.configuration.boot.tmpOnTmpfs = true;
     nixos.configuration.services.nginx.enable = true;
     nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
+    nixos.configuration.services.nscd.enable = false;
+    nixos.configuration.system.nssModules = lib.mkForce [];
     nixos.configuration.systemd.services.nginx.serviceConfig.AmbientCapabilities = 
       lib.mkForce [ "CAP_NET_BIND_SERVICE" ];
     service.useHostStore = true;

examples/minimal/arion-compose.nix

@@ -4,6 +4,7 @@
   config.services = {
 
     webserver = {
+      image.enableRecommendedContents = true;
       service.useHostStore = true;
       service.command = [ "sh" "-c" ''
                   cd "$$WEB_ROOT"

nix/ci.nix

@@ -5,26 +5,20 @@ let
 in
 
 dimension "Nixpkgs version" {
-    "nixos-19_03" = {
-      # flyingcircus.io latest long-term support is based off 19.03
-      # https://flyingcircus.io/doc/
-      # It is nice to have some level of support for their platform,
-      # but we don't guarantee any support.
-      nixpkgsSource = "nixos-19.03";
-      enableDoc = false;
-      nixosTestIsPerl = true;
-    };
     "nixos-20_09" = {
       nixpkgsSource = "nixos-20.09";
       isReferenceNixpkgs = true;
       enableDoc = true;
+      dockerSupportsSystemd = true;
+      nixosHasPodmanDockerSocket = false;
     };
     "nixos-unstable" = {
       nixpkgsSource = "nixos-unstable";
       enableDoc = true;
     };
   } (
-    _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true, nixosTestIsPerl ? false }:
+    _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true,
+             dockerSupportsSystemd ? false, nixosHasPodmanDockerSocket ? true }:
 
 
       dimension "System" {
@@ -34,7 +28,7 @@ dimension "Nixpkgs version" {
         system: { isReferenceTarget ? false, enableNixOSTests ? true }:
           let
             pkgs = import ./. {
-              inherit system nixosTestIsPerl;
+              inherit system dockerSupportsSystemd nixosHasPodmanDockerSocket;
               nixpkgsSrc = sources.${nixpkgsSource};
             };
           in

nix/default.nix

@@ -2,7 +2,8 @@
 , nixpkgsName ? "nixos-unstable"
 , nixpkgsSrc ? sources.${nixpkgsName}
 , system ? builtins.currentSystem
-, nixosTestIsPerl ? false
+, dockerSupportsSystemd ? false
+, nixosHasPodmanDockerSocket ? true
 , ...
 }:
 
@@ -11,8 +12,11 @@ import nixpkgsSrc ({
   config = {
   };
   overlays = [
-    # all the packages are defined there:
+    (_: _: {
-    (_: _: { inherit nixosTestIsPerl; })
+      arionTestingFlags = {
+        inherit dockerSupportsSystemd nixosHasPodmanDockerSocket;
+      };
+    })
     (import ./overlay.nix)
   ];
   inherit system;

nix/sources.json

@@ -11,18 +11,6 @@
         "url": "https://github.com/nmattia/niv/archive/fad2a6cbfb2e7cdebb7cb0ad2f5cc91e2c9bc06b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "nixos-19.03": {
-        "branch": "nixos-19.03",
-        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
-        "homepage": "https://github.com/NixOS/nixpkgs",
-        "owner": "NixOS",
-        "repo": "nixpkgs-channels",
-        "rev": "34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59",
-        "sha256": "11z6ajj108fy2q5g8y4higlcaqncrbjm3dnv17pvif6avagw4mcb",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "nixos-20.09": {
         "branch": "nixos-20.09",
         "description": "Nix Packages collection",
@@ -36,15 +24,15 @@
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-unstable": {
-        "branch": "nixos-unstable",
+        "branch": "master",
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "296793637b22bdb4d23b479879eba0a71c132a66",
+        "rev": "97c3d70a39070547a8342f7ee6f5c4a560282179",
-        "sha256": "0j09yih9693w5vjx64ikfxyja1ha7pisygrwrpg3wfz3sssglg69",
+        "sha256": "1pkagmf42n3v4bjk8jr23hcwpa5qy21w0psi0jbdrbsgpp6rchqa",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/296793637b22bdb4d23b479879eba0a71c132a66.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/97c3d70a39070547a8342f7ee6f5c4a560282179.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
         "version": ""
     },

nix/sources.nix

@@ -6,52 +6,63 @@ let
   # The fetchers. fetch_<type> fetches specs of type <type>.
   #
 
-  fetch_file = pkgs: spec:
+  fetch_file = pkgs: name: spec:
-    if spec.builtin or true then
+    let
-      builtins_fetchurl { inherit (spec) url sha256; }
+      name' = sanitizeName name + "-src";
-    else
+    in
-      pkgs.fetchurl { inherit (spec) url sha256; };
+      if spec.builtin or true then
+        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      else
+        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
 
-  fetch_tarball = pkgs: spec:
+  fetch_tarball = pkgs: name: spec:
-    if spec.builtin or true then
+    let
-      builtins_fetchTarball { inherit (spec) url sha256; }
+      name' = sanitizeName name + "-src";
-    else
+    in
-      pkgs.fetchzip { inherit (spec) url sha256; };
+      if spec.builtin or true then
+        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      else
+        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
 
-  fetch_git = spec:
+  fetch_git = name: spec:
-    builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+    let
+      ref =
+        if spec ? ref then spec.ref else
+          if spec ? branch then "refs/heads/${spec.branch}" else
+            if spec ? tag then "refs/tags/${spec.tag}" else
+              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+    in
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
 
-  fetch_builtin-tarball = spec:
+  fetch_local = spec: spec.path;
-    builtins.trace
-      ''
-        WARNING:
-          The niv type "builtin-tarball" will soon be deprecated. You should
-          instead use `builtin = true`.
 
-          $ niv modify <package> -a type=tarball -a builtin=true
+  fetch_builtin-tarball = name: throw
-      ''
+    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
-      builtins_fetchTarball { inherit (spec) url sha256; };
+        $ niv modify ${name} -a type=tarball -a builtin=true'';
 
-  fetch_builtin-url = spec:
+  fetch_builtin-url = name: throw
-    builtins.trace
+    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
-      ''
+        $ niv modify ${name} -a type=file -a builtin=true'';
-        WARNING:
-          The niv type "builtin-url" will soon be deprecated. You should
-          instead use `builtin = true`.
-
-          $ niv modify <package> -a type=file -a builtin=true
-      ''
-      (builtins_fetchurl { inherit (spec) url sha256; });
 
   #
   # Various helpers
   #
 
+  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
+  sanitizeName = name:
+    (
+      concatMapStrings (s: if builtins.isList s then "-" else s)
+        (
+          builtins.split "[^[:alnum:]+._?=-]+"
+            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
+        )
+    );
+
   # The set of packages used when specs are fetched using non-builtins.
-  mkPkgs = sources:
+  mkPkgs = sources: system:
     let
       sourcesNixpkgs =
-        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
       hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
       hasThisAsNixpkgsPath = <nixpkgs> == ./.;
     in
@@ -71,14 +82,24 @@ let
 
     if ! builtins.hasAttr "type" spec then
       abort "ERROR: niv spec ${name} does not have a 'type' attribute"
-    else if spec.type == "file" then fetch_file pkgs spec
+    else if spec.type == "file" then fetch_file pkgs name spec
-    else if spec.type == "tarball" then fetch_tarball pkgs spec
+    else if spec.type == "tarball" then fetch_tarball pkgs name spec
-    else if spec.type == "git" then fetch_git spec
+    else if spec.type == "git" then fetch_git name spec
-    else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
+    else if spec.type == "local" then fetch_local spec
-    else if spec.type == "builtin-url" then fetch_builtin-url spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then fetch_builtin-url name
     else
       abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
 
+  # If the environment variable NIV_OVERRIDE_${name} is set, then use
+  # the path directly as opposed to the fetched source.
+  replace = name: drv:
+    let
+      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
+    in
+      if ersatz == "" then drv else ersatz;
+
   # Ports of functions for older nix versions
 
   # a Nix version of mapAttrs if the built-in doesn't exist
@@ -87,23 +108,37 @@ let
     listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
   );
 
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatMapStrings = f: list: concatStrings (map f list);
+  concatStrings = builtins.concatStringsSep "";
+
+  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
+  optionalAttrs = cond: as: if cond then as else {};
+
   # fetchTarball version that is compatible between all the versions of Nix
-  builtins_fetchTarball = { url, sha256 }@attrs:
+  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchTarball;
     in
       if lessThan nixVersion "1.12" then
-        fetchTarball { inherit url; }
+        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
       else
         fetchTarball attrs;
 
   # fetchurl version that is compatible between all the versions of Nix
-  builtins_fetchurl = { url, sha256 }@attrs:
+  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchurl;
     in
       if lessThan nixVersion "1.12" then
-        fetchurl { inherit url; }
+        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
       else
         fetchurl attrs;
 
@@ -115,14 +150,15 @@ let
         then abort
           "The values in sources.json should not have an 'outPath' attribute"
         else
-          spec // { outPath = fetch config.pkgs name spec; }
+          spec // { outPath = replace name (fetch config.pkgs name spec); }
     ) config.sources;
 
   # The "config" used by the fetchers
   mkConfig =
-    { sourcesFile ? ./sources.json
+    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
-    , sources ? builtins.fromJSON (builtins.readFile sourcesFile)
+    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
-    , pkgs ? mkPkgs sources
+    , system ? builtins.currentSystem
+    , pkgs ? mkPkgs sources system
     }: rec {
       # The sources, i.e. the attribute set of spec name to spec
       inherit sources;
@@ -130,5 +166,6 @@ let
       # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
       inherit pkgs;
     };
+
 in
 mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

src/haskell/lib/Arion/Images.hs

@@ -1,7 +1,7 @@
 {-# LANGUAGE DeriveGeneric #-}
 {-# LANGUAGE DeriveAnyClass #-}
 {-# LANGUAGE OverloadedStrings #-}
-module Arion.Images 
+module Arion.Images
   ( loadImages
   ) where
 
@@ -22,24 +22,28 @@ loadImages requestedImages = do
   loaded <- getDockerImages
 
   let
-    isNew i = (imageName i <> ":" <> imageTag i) `notElem` loaded
+    isNew i =
+      -- On docker, the image name is unmodified
+      (imageName i <> ":" <> imageTag i) `notElem` loaded
+      -- -- On podman, you automatically get a localhost prefix
+        && ("localhost/" <> imageName i <> ":" <> imageTag i) `notElem` loaded
 
   traverse_ loadImage . filter isNew $ requestedImages
 
 loadImage :: Image -> IO ()
-loadImage (Image { image = Just imgPath, imageName = name }) =
+loadImage Image { image = Just imgPath, imageName = name } =
   withFile (toS imgPath) ReadMode $ \fileHandle -> do
   let procSpec = (Process.proc "docker" [ "load" ]) {
           Process.std_in = Process.UseHandle fileHandle
         }
   Process.withCreateProcess procSpec $ \_in _out _err procHandle -> do
-    e <- Process.waitForProcess procHandle 
+    e <- Process.waitForProcess procHandle
     case e of
       ExitSuccess -> pass
       ExitFailure code ->
         panic $ "docker load failed with exit code " <> show code <> " for image " <> name <> " from path " <> imgPath
 
-loadImage (Image { imageExe = Just imgExe, imageName = name }) = do
+loadImage Image { imageExe = Just imgExe, imageName = name } = do
   let loadSpec = (Process.proc "docker" [ "load" ]) { Process.std_in = Process.CreatePipe }
   Process.withCreateProcess loadSpec $ \(Just inHandle) _out _err loadProcHandle -> do
     let streamSpec = Process.proc (toS imgExe) []
@@ -57,11 +61,11 @@ loadImage (Image { imageExe = Just imgExe, imageName = name }) = do
             _ -> pass
           pass
 
-loadImage (Image { imageName = name }) = do
+loadImage Image { imageName = name } = do
   panic $ "image " <> name <> " doesn't specify an image file or imageExe executable"
 
 
 getDockerImages :: IO [TaggedImage]
 getDockerImages = do
   let procSpec = Process.proc "docker" [ "images",  "--filter", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}" ]
-  (map toS . T.lines . toS) <$> Process.readCreateProcess procSpec ""
+  map toS . T.lines . toS <$> Process.readCreateProcess procSpec ""

src/haskell/testdata/Arion/NixSpec/arion-compose.json

@@ -2,14 +2,14 @@
     "services": {
         "webserver": {
             "command": [
-                "/nix/store/b9w61w4g8sqgrm3rid6ca22krslqghb3-nixos-system-unnamed-19.03.173100.e726e8291b2/init"
+                "/usr/sbin/init"
             ],
             "environment": {
                 "NIX_REMOTE": "",
                 "PATH": "/usr/bin:/run/current-system/sw/bin/",
                 "container": "docker"
             },
-            "image": "arion-base:<HASH>",
+            "image": "webserver:<HASH>",
             "ports": [
                 "8000:80"
             ],
@@ -23,8 +23,7 @@
             "tty": true,
             "volumes": [
                 "/sys/fs/cgroup:/sys/fs/cgroup:ro",
-                "/nix/store:/nix/store:ro",
+                "/nix/store:/nix/store:ro"
-                "/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system:ro"
             ]
         }
     },
@@ -32,8 +31,8 @@
     "x-arion": {
         "images": [
             {
-                "image": "<STOREPATH>",
+                "imageExe": "<STOREPATH>",
-                "imageName": "arion-base",
+                "imageName": "webserver",
                 "imageTag": "<HASH>"
             }
         ],

src/nix/modules.nix

@@ -3,6 +3,5 @@
     ./modules/composition/host-environment.nix
     ./modules/composition/images.nix
     ./modules/composition/service-info.nix
-    ./modules/composition/arion-base-image.nix
     ./modules/composition/composition.nix
 ]

src/nix/modules/composition/arion-base-image.nix

@@ -1,41 +0,0 @@
-
-
-# This module is subject to change.
-# In particular, arion-base should use a generic non-service image building system
-
-{ config, lib, pkgs, ... }: 
-
-let
-
-  tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath));
-  name = "arion-base";
-
-  builtImage = pkgs.dockerTools.buildImage {
-    inherit name;
-    contents = pkgs.runCommand "minimal-contents" {} ''
-        mkdir -p $out/bin $out/usr/bin
-        ln -s /run/system/bin/sh $out/bin/sh
-        ln -s /run/system/usr/bin/env $out/usr/bin/env
-      '';
-    config = {};
-  };
-
-in
-
-{
-
-  options = {
-    arionBaseImage = lib.mkOption {
-      type = lib.types.str;
-      description = "Image to use when using useHostStore. Don't use this option yourself. It's going away.";
-      internal = true;
-    };
-  };
-
-  config = {
-    arionBaseImage = "${name}:${tag}";
-    build.imagesToLoad = lib.mkIf (lib.any (s: s.service.useHostStore) (lib.attrValues config.services)) [
-      { image = builtImage; imageName = name; imageTag = tag; }
-    ];  
-  };
-}

src/nix/modules/lib/assert.nix

@@ -3,14 +3,15 @@
 # based on nixpkgs/nixos/modules/system/activation/top-level.nix
 
 let
-  inherit (lib) filter concatStringsSep types mkOption;
+  inherit (lib)
-
+    concatStringsSep
-  # lib.showWarnings since 19.09
+    filter
-  showWarnings = warnings: res: lib.fold (w: x: lib.warn w x) res warnings;
+    mkOption
-  warn = msg: builtins.trace "warning: ${msg}";
+    showWarnings
+    types
+    ;
 
   # Handle assertions and warnings
-
   failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions);
 
   assertWarn = if failedAssertions != []

src/nix/modules/service/all-modules.nix

@@ -5,7 +5,9 @@
   ./host-store.nix
   ./context.nix
   ./image.nix
+  ./image-recommended.nix
   ./nixos.nix
   ./nixos-init.nix
   ../lib/assert.nix
+  ./check-sys_admin.nix
 ]

src/nix/modules/service/check-sys_admin.nix

@@ -0,0 +1,30 @@
+{ config, lib, name, ... }:
+let
+  inherit (lib)
+    concatStringsSep
+    optional
+    ;
+
+  dynamicUserServices = lib.attrNames (
+    lib.filterAttrs
+      (k: v:
+        v.enable &&
+        v.serviceConfig.DynamicUser or false)
+      config.nixos.evaluatedConfig.systemd.services
+    );
+
+  
+in
+{
+  config = {
+    warnings = 
+      optional (config.nixos.useSystemd && !(config.service.capabilities.SYS_ADMIN or false) && dynamicUserServices != []) (
+        ''In service ${name}, the following units require `SYS_ADMIN` capability
+          because of DynamicUser.
+          ${concatStringsSep "\n" (map (srv: "    - services.${name}.nixos.configuration.systemd.services.${srv}") dynamicUserServices)}
+          You can avoid DynamicUser or use
+              services.${name}.service.capabilities.SYS_ADMIN = true;
+        ''
+      );
+  };
+}

src/nix/modules/service/host-store.nix

@@ -29,12 +29,10 @@ in
     };
   };
   config = mkIf config.service.useHostStore {
-    image.nixBuild = false; # no need to build and load
+    image.includeStorePaths = false;
-    service.image = config.composition.arionBaseImage;
     service.environment.NIX_REMOTE = lib.optionalString config.service.useHostNixDaemon "daemon";
     service.volumes = [
       "${config.host.nixStorePrefix}/nix/store:/nix/store${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"
-      "${config.host.nixStorePrefix}${pkgs.buildEnv { name = "container-system-env"; paths = [ pkgs.bashInteractive pkgs.coreutils ]; }}:/run/system${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"
       ] ++ lib.optional config.service.useHostNixDaemon "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket";
     service.command = lib.mkDefault (map escape (config.image.rawConfig.Cmd or []));
   };

src/nix/modules/service/image-recommended.nix

@@ -0,0 +1,36 @@
+{ config, lib, pkgs, ... }:
+let
+  inherit (lib)
+    mkIf
+    mkOption
+    types
+    ;
+  inherit (types)
+    bool
+    ;
+
+  recommendedContents = { runCommand, bash, coreutils }:
+    runCommand "recommended-contents" {} ''
+      mkdir -p $out/bin $out/usr/bin $out/var/empty
+      ln -s ${bash}/bin/sh $out/bin/sh
+      ln -s ${coreutils}/bin/env $out/usr/bin/env
+    '';
+in
+{
+  options = {
+    image.enableRecommendedContents = mkOption {
+      type = bool;
+      default = false;
+      description = ''
+        Add the `/bin/sh` and `/usr/bin/env` symlinks and some lightweight
+        files.
+      '';
+    };
+  };
+
+  config = {
+    image.contents = mkIf config.image.enableRecommendedContents [
+      (pkgs.callPackage recommendedContents {})
+    ];
+  };
+}

src/nix/modules/service/image.nix

@@ -1,6 +1,15 @@
 { pkgs, lib, config, options, ... }:
 let
-  inherit (lib) types mkOption;
+  inherit (lib)
+    functionArgs
+    mkOption
+    optionalAttrs
+    types
+    warn
+    ;
+  inherit (pkgs)
+    dockerTools
+    ;
   inherit (types) attrsOf listOf nullOr package str unspecified bool;
 
   # TODO: dummy-config is a useless layer. Nix 2.3 will let us inspect
@@ -9,15 +18,37 @@ let
     (pkgs.writeText "dummy-config.json" (builtins.toJSON config.image.rawConfig))
   ];
 
+  includeStorePathsWarningAndDefault = lib.warn ''
+    You're using a version of Nixpkgs that doesn't support the includeStorePaths
+    parameter in dockerTools.streamLayeredImage. Without this, Arion's
+    useHostStore does not achieve the intended speedup.
+  '' {};
+
   buildOrStreamLayeredImage = args:
-    if pkgs.dockerTools?streamLayeredImage
+    let
-    then pkgs.dockerTools.streamLayeredImage args // { isExe = true; }
+      args_base = builtins.intersectAttrs
-    else pkgs.dockerTools.buildLayeredImage args;
+        {
+          name = null; tag = null; contents = null; config = null;
+          created = null; extraCommands = null; maxLayers = null;
+        }
+        args;
+      acceptedArgs = functionArgs dockerTools.streamLayeredImage;
+      args_no_store = lib.optionalAttrs (!(args.includeStorePaths or true)) (
+        if acceptedArgs ? includeStorePaths
+        then { inherit (args) includeStorePaths; }
+        else includeStorePathsWarningAndDefault
+      );
+      args_streamLayered = args_base // args_no_store;
+    in
+      if dockerTools?streamLayeredImage
+      then dockerTools.streamLayeredImage args_streamLayered // { isExe = true; }
+      else dockerTools.buildLayeredImage args_base;
 
   builtImage = buildOrStreamLayeredImage {
     inherit (config.image)
       name
       contents
+      includeStorePaths
       ;
     config = config.image.rawConfig;
     maxLayers = 100;
@@ -89,6 +120,15 @@ in
          Top level paths in the container.
       '';
     };
+    image.includeStorePaths = mkOption {
+      type = bool;
+      default = true;
+      internal = true;
+      description = ''
+        Include all referenced store paths. You generally want this in your
+        image, unless you load store paths via some other means, like useHostStore = true;
+      '';
+    };
     image.rawConfig = mkOption {
       type = attrsOf unspecified;
       default = {};

src/nix/modules/service/nixos-init.nix

@@ -24,7 +24,13 @@ in
       ../nixos/default-shell.nix
       (pkgs.path + "/nixos/modules/profiles/minimal.nix")
     ];
-    image.command = [ "${config.nixos.build.toplevel}/init" ];
+    image.command = [ "/usr/sbin/init" ];
+    image.contents = [
+      (pkgs.runCommand "root-init" {} ''
+        mkdir -p $out/usr/sbin
+        ln -s ${config.nixos.build.toplevel}/init $out/usr/sbin/init
+      '')
+    ];
     service.environment.container = "docker";
     service.environment.PATH = "/usr/bin:/run/current-system/sw/bin/";
     service.volumes = [

tests/arion-test-perl/README.md

@@ -1,4 +0,0 @@
-
-This test suite exists only to keep tests around for older versions of NixOS.
-
-This will be removed when 19.09 becomes irrelevant.

tests/arion-test-perl/default.nix

@@ -1,60 +0,0 @@
-{ pkgs, ... }:
-
-let
-  # To make some prebuilt derivations available in the vm
-  preEval = modules: import ../../src/nix/eval-composition.nix {
-    inherit modules;
-    inherit pkgs;
-  };
-in
-{
-  name = "arion-test";
-  machine = { pkgs, lib, ... }: {
-    environment.systemPackages = [
-      pkgs.arion
-    ];
-    virtualisation.docker.enable = true;
-    
-    # no caches, because no internet
-    nix.binaryCaches = lib.mkForce [];
-
-    # FIXME: Sandbox seems broken with current version of NixOS test
-    #        w/ writable store. Error:
-    #        machine# error: linking '/nix/store/7r8z2zvhwda85pgpdn5hzzz6hs1njklc-stdenv-linux.drv.chroot/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh' to '/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh': Operation not permitted
-    #
-    #        There should be no reason why arion can't run without
-    #        sandboxing, so please re-enable.
-    nix.useSandbox = false;
-
-    virtualisation.writableStore = true;
-    virtualisation.pathsInNixDB = [
-      # Pre-build the image because we don't want to build the world
-      # in the vm.
-      (preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml
-      (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml
-      (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml
-      pkgs.stdenv
-    ];
-
-    virtualisation.memorySize = 512;
-  };
-  testScript = ''
-    $machine->fail("curl localhost:8000");
-    $machine->succeed("docker --version");
-
-    my $makeSubtest = sub {
-      my ( $subtestName, $exampleSrc ) = @_;
-
-      subtest $subtestName => sub {
-        $machine->succeed("rm -rf work && cp -frT $exampleSrc work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
-        $machine->waitUntilSucceeds("curl localhost:8000");
-        $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down");
-        $machine->waitUntilFails("curl localhost:8000");
-      };
-    };
-
-    $makeSubtest->("minimal", "${../../examples/minimal}");
-    $makeSubtest->("full-nixos", "${../../examples/full-nixos}");
-    $makeSubtest->("nixos-unit", "${../../examples/nixos-unit}");
-  '';
-}

tests/arion-test/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ usePodman ? false, pkgs, lib, ... }:
 
 let
   # To make some prebuilt derivations available in the vm
@@ -6,14 +6,26 @@ let
     inherit modules;
     inherit pkgs;
   };
+
+  inherit (lib)
+    optionalAttrs
+    optionalString
+    ;
+
+  haveSystemd = usePodman || pkgs.arionTestingFlags.dockerSupportsSystemd;
+
 in
 {
   name = "arion-test";
   machine = { pkgs, lib, ... }: {
     environment.systemPackages = [
       pkgs.arion
-    ];
+    ] ++ lib.optional usePodman pkgs.docker;
-    virtualisation.docker.enable = true;
+    virtualisation.docker.enable = !usePodman;
+    virtualisation.podman = optionalAttrs usePodman {
+      enable = true;
+      dockerSocket.enable = true;
+    };
     
     # no caches, because no internet
     nix.binaryCaches = lib.mkForce [];
@@ -38,6 +50,7 @@ in
     ];
 
     virtualisation.memorySize = 1024;
+    virtualisation.diskSize = 8000;
   };
   testScript = ''
     machine.fail("curl --fail localhost:8000")
@@ -57,6 +70,44 @@ in
         )
         machine.wait_until_fails("curl --fail localhost:8000")
 
+    # Tests
+    #  - running same image again doesn't require a `docker load`
+    with subtest("docker load only once"):
+        # We assume image loading relies on the `docker images` and `docker load` commands, so this should fail
+        machine.fail(
+            "export REAL_DOCKER=$(which docker); rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' PATH=\"${pkgs.writeScriptBin "docker" ''
+              #!${pkgs.runtimeShell} -eu
+              echo 1>&2 "This failure is expected. Args were" "$@"
+              echo "$@" >/tmp/docker-args
+              exit 1
+            ''}/bin:$PATH\" arion up -d"
+        )
+        machine.succeed(
+            "export REAL_DOCKER=$(which docker); rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' PATH=\"${pkgs.writeScriptBin "docker" ''
+              #!${pkgs.runtimeShell} -eu
+              case $1 in
+                load)
+                  echo 1>&2 "arion must not docker load when upping the same deployment for the second time"
+                  exit 1
+                  ;;
+                images)
+                  echo 1>&2 "execing docker to list images"
+                  exec $REAL_DOCKER "$@"
+                  ;;
+                *)
+                  echo 1>&2 "Unknown docker invocation. This may be a shortcoming of this docker mock."
+                  echo 1>&2 "Invocation: docker" "$@"
+                  ;;
+              esac
+            ''}/bin:$PATH\" arion up -d"
+        )
+        machine.wait_until_succeeds("curl --fail localhost:8000")
+        machine.succeed(
+            "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
+        )
+        machine.wait_until_fails("curl --fail localhost:8000")
+
+
     # Tests
     #  - examples/flake
     # This _test_ doesn't work because flake-compat fetches the github
@@ -71,6 +122,7 @@ in
     #     machine.succeed("cd work && NIX_PATH= arion down")
     #     machine.wait_until_fails("curl --fail localhost:8000")
 
+    ${optionalString haveSystemd ''
     # Tests
     #  - arion exec
     #  - examples/full-nixos
@@ -95,6 +147,7 @@ in
             "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
         )
         machine.wait_until_fails("curl --fail localhost:8000")
+    ''}
 
     # Tests
     #  - examples/nixos-unit

tests/default.nix

@@ -1,11 +1,16 @@
-{ pkgs ? import ../pkgs.nix, nixosTestIsPerl ? false }:
+{ pkgs ? import ../pkgs.nix, arionTestingFlags ? {} }:
 let
   inherit (pkgs) nixosTest recurseIntoAttrs arion;
 in
 
 recurseIntoAttrs {
 
-  test = if nixosTestIsPerl then nixosTest ./arion-test-perl else nixosTest ./arion-test;
+  test = nixosTest ./arion-test;
+
+  testWithPodman =
+    if arionTestingFlags.nixosHasPodmanDockerSocket
+    then nixosTest (pkgs.callPackage ./arion-test { usePodman = true; })
+    else {};
 
   testBuild = arion.build {