ghoscht/arion
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #117 from hercules-ci/nixos-21.05-podman-preparation

Browse source 
NixOS 21.05/podman preparation
This commit is contained in:
Robert Hensing 2021-05-31 17:06:34 +02:00 committed by GitHub
commit 865055787a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 360 additions and 219 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
CHANGELOG.md
View file

@ -2,13 +2,28 @@
## Next
### Changed
* `useHostStore` now uses an image derived from the `image.*` options. You may
need to enable `enableRecommendedContents` because with this change, files
like `/bin/sh` aren't added by default anymore.
* Drop obsolete NixOS 19.03, 19.09 and 20.03 from CI.
### Added
* NixOS-based containers can now run on Podman when it is configured to provide a docker socket. See the [installation docs](https://docs.hercules-ci.com/arion/#_nixos).
* Support `service.dns`, for overriding the DNS servers used by containers.
* Support `service.labels`, which is useful for autodiscovery among other things.
* Add a tested example for Traefik with label-based routing.
* Drop obsolete NixOS 19.09 and 20.03 support. It may still be usable there.
* Add a `flake.nix` and an experimental flake example
* Add a warning when systemd `DynamicUser` is used but not available to the
container.
## 0.1.2.0 -- 2020-03-05

21
docs/modules/ROOT/pages/index.adoc
View file

@ -39,6 +39,12 @@ Arion allows to compose containers with different granularity:
* <<NixOS: run full OS>>
* <<Docker image from DockerHub>>
Full NixOS is supported on
* docker-compose + podman with docker socket (NixOS >= 21.05)
* docker-compose + docker, before cgroupsv2 (NixOS < 21.05)
`podman-compose` support is currently WIP on a separate branch.
== Installation
=== Nix
@ -52,10 +58,17 @@ $ nix-env -iA arion -f https://github.com/hercules-ci/arion/tarball/master
Add this module to your NixOS configuration:
```nix
{ ... }: {
environment.systemPackages = [ (import (builtins.fetchTarball https://github.com/hercules-ci/arion/tarball/master) {}).arion ];
virtualisation.docker.enable = true;
users.extraUsers.myuser.extraGroups = ["docker"];
{ pkgs, ... }: {
environment.systemPackages = [
pkgs.arion
pkgs.docker # docker CLI will use podman socket
];
virtualisation.docker.enable = false;
virtualisation.podman.enable = true;
virtualisation.podman.dockerSocket.enable = true;
# Use your username instead of `myuser`
users.extraUsers.myuser.extraGroups = ["podman"];
}
```

19
docs/modules/ROOT/partials/NixOSOptions.adoc
View file

@ -197,6 +197,25 @@ Default::
----
No Example:: {blank}
== services.<name>.image.enableRecommendedContents
Add the `/bin/sh` and `/usr/bin/env` symlinks and some lightweight
files.
[discrete]
=== details
Type:: boolean
Default::
+
----
false
----
No Example:: {blank}
== services.<name>.image.name

2
examples/full-nixos/arion-compose.nix
View file

@ -4,6 +4,8 @@
nixos.configuration.boot.tmpOnTmpfs = true;
nixos.configuration.services.nginx.enable = true;
nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
nixos.configuration.services.nscd.enable = false;
nixos.configuration.system.nssModules = lib.mkForce [];
nixos.configuration.systemd.services.nginx.serviceConfig.AmbientCapabilities =
lib.mkForce [ "CAP_NET_BIND_SERVICE" ];
service.useHostStore = true;

1
examples/minimal/arion-compose.nix
View file

@ -4,6 +4,7 @@
config.services = {
webserver = {
image.enableRecommendedContents = true;
service.useHostStore = true;
service.command = [ "sh" "-c" ''
cd "$$WEB_ROOT"

16
nix/ci.nix
View file

@ -5,26 +5,20 @@ let
in
dimension "Nixpkgs version" {
"nixos-19_03" = {
# flyingcircus.io latest long-term support is based off 19.03
# https://flyingcircus.io/doc/
# It is nice to have some level of support for their platform,
# but we don't guarantee any support.
nixpkgsSource = "nixos-19.03";
enableDoc = false;
nixosTestIsPerl = true;
};
"nixos-20_09" = {
nixpkgsSource = "nixos-20.09";
isReferenceNixpkgs = true;
enableDoc = true;
dockerSupportsSystemd = true;
nixosHasPodmanDockerSocket = false;
};
"nixos-unstable" = {
nixpkgsSource = "nixos-unstable";
enableDoc = true;
};
} (
_name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true, nixosTestIsPerl ? false }:
_name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true,
dockerSupportsSystemd ? false, nixosHasPodmanDockerSocket ? true }:
dimension "System" {
@ -34,7 +28,7 @@ dimension "Nixpkgs version" {
system: { isReferenceTarget ? false, enableNixOSTests ? true }:
let
pkgs = import ./. {
inherit system nixosTestIsPerl;
inherit system dockerSupportsSystemd nixosHasPodmanDockerSocket;
nixpkgsSrc = sources.${nixpkgsSource};
};
in

10
nix/default.nix
View file

@ -2,7 +2,8 @@
, nixpkgsName ? "nixos-unstable"
, nixpkgsSrc ? sources.${nixpkgsName}
, system ? builtins.currentSystem
, nixosTestIsPerl ? false
, dockerSupportsSystemd ? false
, nixosHasPodmanDockerSocket ? true
, ...
}:
@ -11,8 +12,11 @@ import nixpkgsSrc ({
config = {
};
overlays = [
# all the packages are defined there:
(_: _: { inherit nixosTestIsPerl; })
(_: _: {
arionTestingFlags = {
inherit dockerSupportsSystemd nixosHasPodmanDockerSocket;
};
})
(import ./overlay.nix)
];
inherit system;

20
nix/sources.json
View file

@ -11,18 +11,6 @@
"url": "https://github.com/nmattia/niv/archive/fad2a6cbfb2e7cdebb7cb0ad2f5cc91e2c9bc06b.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixos-19.03": {
"branch": "nixos-19.03",
"description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
"homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS",
"repo": "nixpkgs-channels",
"rev": "34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59",
"sha256": "11z6ajj108fy2q5g8y4higlcaqncrbjm3dnv17pvif6avagw4mcb",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixos-20.09": {
"branch": "nixos-20.09",
"description": "Nix Packages collection",
@ -36,15 +24,15 @@
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixos-unstable": {
"branch": "nixos-unstable",
"branch": "master",
"description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
"homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "296793637b22bdb4d23b479879eba0a71c132a66",
"sha256": "0j09yih9693w5vjx64ikfxyja1ha7pisygrwrpg3wfz3sssglg69",
"rev": "97c3d70a39070547a8342f7ee6f5c4a560282179",
"sha256": "1pkagmf42n3v4bjk8jr23hcwpa5qy21w0psi0jbdrbsgpp6rchqa",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/296793637b22bdb4d23b479879eba0a71c132a66.tar.gz",
"url": "https://github.com/NixOS/nixpkgs/archive/97c3d70a39070547a8342f7ee6f5c4a560282179.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
"version": ""
},

129
nix/sources.nix
View file

@ -6,52 +6,63 @@ let
# The fetchers. fetch_<type> fetches specs of type <type>.
#
fetch_file = pkgs: spec:
if spec.builtin or true then
builtins_fetchurl { inherit (spec) url sha256; }
else
pkgs.fetchurl { inherit (spec) url sha256; };
fetch_file = pkgs: name: spec:
let
name' = sanitizeName name + "-src";
in
if spec.builtin or true then
builtins_fetchurl { inherit (spec) url sha256; name = name'; }
else
pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
fetch_tarball = pkgs: spec:
if spec.builtin or true then
builtins_fetchTarball { inherit (spec) url sha256; }
else
pkgs.fetchzip { inherit (spec) url sha256; };
fetch_tarball = pkgs: name: spec:
let
name' = sanitizeName name + "-src";
in
if spec.builtin or true then
builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
else
pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
fetch_git = spec:
builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
fetch_git = name: spec:
let
ref =
if spec ? ref then spec.ref else
if spec ? branch then "refs/heads/${spec.branch}" else
if spec ? tag then "refs/tags/${spec.tag}" else
abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
in
builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
fetch_builtin-tarball = spec:
builtins.trace
''
WARNING:
The niv type "builtin-tarball" will soon be deprecated. You should
instead use `builtin = true`.
fetch_local = spec: spec.path;
$ niv modify <package> -a type=tarball -a builtin=true
''
builtins_fetchTarball { inherit (spec) url sha256; };
fetch_builtin-tarball = name: throw
''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
$ niv modify ${name} -a type=tarball -a builtin=true'';
fetch_builtin-url = spec:
builtins.trace
''
WARNING:
The niv type "builtin-url" will soon be deprecated. You should
instead use `builtin = true`.
$ niv modify <package> -a type=file -a builtin=true
''
(builtins_fetchurl { inherit (spec) url sha256; });
fetch_builtin-url = name: throw
''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
$ niv modify ${name} -a type=file -a builtin=true'';
#
# Various helpers
#
# https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
sanitizeName = name:
(
concatMapStrings (s: if builtins.isList s then "-" else s)
(
builtins.split "[^[:alnum:]+._?=-]+"
((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
)
);
# The set of packages used when specs are fetched using non-builtins.
mkPkgs = sources:
mkPkgs = sources: system:
let
sourcesNixpkgs =
import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
hasThisAsNixpkgsPath = <nixpkgs> == ./.;
in
@ -71,14 +82,24 @@ let
if ! builtins.hasAttr "type" spec then
abort "ERROR: niv spec ${name} does not have a 'type' attribute"
else if spec.type == "file" then fetch_file pkgs spec
else if spec.type == "tarball" then fetch_tarball pkgs spec
else if spec.type == "git" then fetch_git spec
else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
else if spec.type == "builtin-url" then fetch_builtin-url spec
else if spec.type == "file" then fetch_file pkgs name spec
else if spec.type == "tarball" then fetch_tarball pkgs name spec
else if spec.type == "git" then fetch_git name spec
else if spec.type == "local" then fetch_local spec
else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
else if spec.type == "builtin-url" then fetch_builtin-url name
else
abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
# If the environment variable NIV_OVERRIDE_${name} is set, then use
# the path directly as opposed to the fetched source.
replace = name: drv:
let
saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
in
if ersatz == "" then drv else ersatz;
# Ports of functions for older nix versions
# a Nix version of mapAttrs if the built-in doesn't exist
@ -87,23 +108,37 @@ let
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
);
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
concatMapStrings = f: list: concatStrings (map f list);
concatStrings = builtins.concatStringsSep "";
# https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
optionalAttrs = cond: as: if cond then as else {};
# fetchTarball version that is compatible between all the versions of Nix
builtins_fetchTarball = { url, sha256 }@attrs:
builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchTarball;
in
if lessThan nixVersion "1.12" then
fetchTarball { inherit url; }
fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
else
fetchTarball attrs;
# fetchurl version that is compatible between all the versions of Nix
builtins_fetchurl = { url, sha256 }@attrs:
builtins_fetchurl = { url, name ? null, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchurl;
in
if lessThan nixVersion "1.12" then
fetchurl { inherit url; }
fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
else
fetchurl attrs;
@ -115,14 +150,15 @@ let
then abort
"The values in sources.json should not have an 'outPath' attribute"
else
spec // { outPath = fetch config.pkgs name spec; }
spec // { outPath = replace name (fetch config.pkgs name spec); }
) config.sources;
# The "config" used by the fetchers
mkConfig =
{ sourcesFile ? ./sources.json
, sources ? builtins.fromJSON (builtins.readFile sourcesFile)
, pkgs ? mkPkgs sources
{ sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
, sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
, system ? builtins.currentSystem
, pkgs ? mkPkgs sources system
}: rec {
# The sources, i.e. the attribute set of spec name to spec
inherit sources;
@ -130,5 +166,6 @@ let
# The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
inherit pkgs;
};
in
mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

18
src/haskell/lib/Arion/Images.hs
View file

@ -1,7 +1,7 @@
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE OverloadedStrings #-}
module Arion.Images
module Arion.Images
( loadImages
) where
@ -22,24 +22,28 @@ loadImages requestedImages = do
loaded <- getDockerImages
let
isNew i = (imageName i <> ":" <> imageTag i) `notElem` loaded
isNew i =
-- On docker, the image name is unmodified
(imageName i <> ":" <> imageTag i) `notElem` loaded
-- -- On podman, you automatically get a localhost prefix
&& ("localhost/" <> imageName i <> ":" <> imageTag i) `notElem` loaded
traverse_ loadImage . filter isNew $ requestedImages
loadImage :: Image -> IO ()
loadImage (Image { image = Just imgPath, imageName = name }) =
loadImage Image { image = Just imgPath, imageName = name } =
withFile (toS imgPath) ReadMode $ \fileHandle -> do
let procSpec = (Process.proc "docker" [ "load" ]) {
Process.std_in = Process.UseHandle fileHandle
}
Process.withCreateProcess procSpec $ \_in _out _err procHandle -> do
e <- Process.waitForProcess procHandle
e <- Process.waitForProcess procHandle
case e of
ExitSuccess -> pass
ExitFailure code ->
panic $ "docker load failed with exit code " <> show code <> " for image " <> name <> " from path " <> imgPath
loadImage (Image { imageExe = Just imgExe, imageName = name }) = do
loadImage Image { imageExe = Just imgExe, imageName = name } = do
let loadSpec = (Process.proc "docker" [ "load" ]) { Process.std_in = Process.CreatePipe }
Process.withCreateProcess loadSpec $ \(Just inHandle) _out _err loadProcHandle -> do
let streamSpec = Process.proc (toS imgExe) []
@ -57,11 +61,11 @@ loadImage (Image { imageExe = Just imgExe, imageName = name }) = do
_ -> pass
pass
loadImage (Image { imageName = name }) = do
loadImage Image { imageName = name } = do
panic $ "image " <> name <> " doesn't specify an image file or imageExe executable"
getDockerImages :: IO [TaggedImage]
getDockerImages = do
let procSpec = Process.proc "docker" [ "images", "--filter", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}" ]
(map toS . T.lines . toS) <$> Process.readCreateProcess procSpec ""
map toS . T.lines . toS <$> Process.readCreateProcess procSpec ""

11
src/haskell/testdata/Arion/NixSpec/arion-compose.json vendored
View file

@ -2,14 +2,14 @@
"services": {
"webserver": {
"command": [
"/nix/store/b9w61w4g8sqgrm3rid6ca22krslqghb3-nixos-system-unnamed-19.03.173100.e726e8291b2/init"
"/usr/sbin/init"
],
"environment": {
"NIX_REMOTE": "",
"PATH": "/usr/bin:/run/current-system/sw/bin/",
"container": "docker"
},
"image": "arion-base:<HASH>",
"image": "webserver:<HASH>",
"ports": [
"8000:80"
],
@ -23,8 +23,7 @@
"tty": true,
"volumes": [
"/sys/fs/cgroup:/sys/fs/cgroup:ro",
"/nix/store:/nix/store:ro",
"/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system:ro"
"/nix/store:/nix/store:ro"
]
}
},
@ -32,8 +31,8 @@
"x-arion": {
"images": [
{
"image": "<STOREPATH>",
"imageName": "arion-base",
"imageExe": "<STOREPATH>",
"imageName": "webserver",
"imageTag": "<HASH>"
}
],

1
src/nix/modules.nix
View file

@ -3,6 +3,5 @@
./modules/composition/host-environment.nix
./modules/composition/images.nix
./modules/composition/service-info.nix
./modules/composition/arion-base-image.nix
./modules/composition/composition.nix
]

41
src/nix/modules/composition/arion-base-image.nix
View file

@ -1,41 +0,0 @@
# This module is subject to change.
# In particular, arion-base should use a generic non-service image building system
{ config, lib, pkgs, ... }:
let
tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath));
name = "arion-base";
builtImage = pkgs.dockerTools.buildImage {
inherit name;
contents = pkgs.runCommand "minimal-contents" {} ''
mkdir -p $out/bin $out/usr/bin
ln -s /run/system/bin/sh $out/bin/sh
ln -s /run/system/usr/bin/env $out/usr/bin/env
'';
config = {};
};
in
{
options = {
arionBaseImage = lib.mkOption {
type = lib.types.str;
description = "Image to use when using useHostStore. Don't use this option yourself. It's going away.";
internal = true;
};
};
config = {
arionBaseImage = "${name}:${tag}";
build.imagesToLoad = lib.mkIf (lib.any (s: s.service.useHostStore) (lib.attrValues config.services)) [
{ image = builtImage; imageName = name; imageTag = tag; }
];
};
}

13
src/nix/modules/lib/assert.nix
View file

@ -3,14 +3,15 @@
# based on nixpkgs/nixos/modules/system/activation/top-level.nix
let
inherit (lib) filter concatStringsSep types mkOption;
# lib.showWarnings since 19.09
showWarnings = warnings: res: lib.fold (w: x: lib.warn w x) res warnings;
warn = msg: builtins.trace "warning: ${msg}";
inherit (lib)
concatStringsSep
filter
mkOption
showWarnings
types
;
# Handle assertions and warnings
failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions);
assertWarn = if failedAssertions != []

2
src/nix/modules/service/all-modules.nix
View file

@ -5,7 +5,9 @@
./host-store.nix
./context.nix
./image.nix
./image-recommended.nix
./nixos.nix
./nixos-init.nix
../lib/assert.nix
./check-sys_admin.nix
]

30
src/nix/modules/service/check-sys_admin.nix Normal file
View file

@ -0,0 +1,30 @@
{ config, lib, name, ... }:
let
inherit (lib)
concatStringsSep
optional
;
dynamicUserServices = lib.attrNames (
lib.filterAttrs
(k: v:
v.enable &&
v.serviceConfig.DynamicUser or false)
config.nixos.evaluatedConfig.systemd.services
);
in
{
config = {
warnings =
optional (config.nixos.useSystemd && !(config.service.capabilities.SYS_ADMIN or false) && dynamicUserServices != []) (
''In service ${name}, the following units require `SYS_ADMIN` capability
because of DynamicUser.
${concatStringsSep "\n" (map (srv: " - services.${name}.nixos.configuration.systemd.services.${srv}") dynamicUserServices)}
You can avoid DynamicUser or use
services.${name}.service.capabilities.SYS_ADMIN = true;
''
);
};
}

4
src/nix/modules/service/host-store.nix
View file

@ -29,12 +29,10 @@ in
};
};
config = mkIf config.service.useHostStore {
image.nixBuild = false; # no need to build and load
service.image = config.composition.arionBaseImage;
image.includeStorePaths = false;
service.environment.NIX_REMOTE = lib.optionalString config.service.useHostNixDaemon "daemon";
service.volumes = [
"${config.host.nixStorePrefix}/nix/store:/nix/store${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"
"${config.host.nixStorePrefix}${pkgs.buildEnv { name = "container-system-env"; paths = [ pkgs.bashInteractive pkgs.coreutils ]; }}:/run/system${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"
] ++ lib.optional config.service.useHostNixDaemon "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket";
service.command = lib.mkDefault (map escape (config.image.rawConfig.Cmd or []));
};

36
src/nix/modules/service/image-recommended.nix Normal file
View file

@ -0,0 +1,36 @@
{ config, lib, pkgs, ... }:
let
inherit (lib)
mkIf
mkOption
types
;
inherit (types)
bool
;
recommendedContents = { runCommand, bash, coreutils }:
runCommand "recommended-contents" {} ''
mkdir -p $out/bin $out/usr/bin $out/var/empty
ln -s ${bash}/bin/sh $out/bin/sh
ln -s ${coreutils}/bin/env $out/usr/bin/env
'';
in
{
options = {
image.enableRecommendedContents = mkOption {
type = bool;
default = false;
description = ''
Add the `/bin/sh` and `/usr/bin/env` symlinks and some lightweight
files.
'';
};
};
config = {
image.contents = mkIf config.image.enableRecommendedContents [
(pkgs.callPackage recommendedContents {})
];
};
}

48
src/nix/modules/service/image.nix
View file

@ -1,6 +1,15 @@
{ pkgs, lib, config, options, ... }:
let
inherit (lib) types mkOption;
inherit (lib)
functionArgs
mkOption
optionalAttrs
types
warn
;
inherit (pkgs)
dockerTools
;
inherit (types) attrsOf listOf nullOr package str unspecified bool;
# TODO: dummy-config is a useless layer. Nix 2.3 will let us inspect
@ -9,15 +18,37 @@ let
(pkgs.writeText "dummy-config.json" (builtins.toJSON config.image.rawConfig))
];
includeStorePathsWarningAndDefault = lib.warn ''
You're using a version of Nixpkgs that doesn't support the includeStorePaths
parameter in dockerTools.streamLayeredImage. Without this, Arion's
useHostStore does not achieve the intended speedup.
'' {};
buildOrStreamLayeredImage = args:
if pkgs.dockerTools?streamLayeredImage
then pkgs.dockerTools.streamLayeredImage args // { isExe = true; }
else pkgs.dockerTools.buildLayeredImage args;
let
args_base = builtins.intersectAttrs
{
name = null; tag = null; contents = null; config = null;
created = null; extraCommands = null; maxLayers = null;
}
args;
acceptedArgs = functionArgs dockerTools.streamLayeredImage;
args_no_store = lib.optionalAttrs (!(args.includeStorePaths or true)) (
if acceptedArgs ? includeStorePaths
then { inherit (args) includeStorePaths; }
else includeStorePathsWarningAndDefault
);
args_streamLayered = args_base // args_no_store;
in
if dockerTools?streamLayeredImage
then dockerTools.streamLayeredImage args_streamLayered // { isExe = true; }
else dockerTools.buildLayeredImage args_base;
builtImage = buildOrStreamLayeredImage {
inherit (config.image)
name
contents
includeStorePaths
;
config = config.image.rawConfig;
maxLayers = 100;
@ -89,6 +120,15 @@ in
Top level paths in the container.
'';
};
image.includeStorePaths = mkOption {
type = bool;
default = true;
internal = true;
description = ''
Include all referenced store paths. You generally want this in your
image, unless you load store paths via some other means, like useHostStore = true;
'';
};
image.rawConfig = mkOption {
type = attrsOf unspecified;
default = {};

8
src/nix/modules/service/nixos-init.nix
View file

@ -24,7 +24,13 @@ in
../nixos/default-shell.nix
(pkgs.path + "/nixos/modules/profiles/minimal.nix")
];
image.command = [ "${config.nixos.build.toplevel}/init" ];
image.command = [ "/usr/sbin/init" ];
image.contents = [
(pkgs.runCommand "root-init" {} ''
mkdir -p $out/usr/sbin
ln -s ${config.nixos.build.toplevel}/init $out/usr/sbin/init
'')
];
service.environment.container = "docker";
service.environment.PATH = "/usr/bin:/run/current-system/sw/bin/";
service.volumes = [

4
tests/arion-test-perl/README.md
View file

@ -1,4 +0,0 @@
This test suite exists only to keep tests around for older versions of NixOS.
This will be removed when 19.09 becomes irrelevant.

60
tests/arion-test-perl/default.nix
View file

@ -1,60 +0,0 @@
{ pkgs, ... }:
let
# To make some prebuilt derivations available in the vm
preEval = modules: import ../../src/nix/eval-composition.nix {
inherit modules;
inherit pkgs;
};
in
{
name = "arion-test";
machine = { pkgs, lib, ... }: {
environment.systemPackages = [
pkgs.arion
];
virtualisation.docker.enable = true;
# no caches, because no internet
nix.binaryCaches = lib.mkForce [];
# FIXME: Sandbox seems broken with current version of NixOS test
# w/ writable store. Error:
# machine# error: linking '/nix/store/7r8z2zvhwda85pgpdn5hzzz6hs1njklc-stdenv-linux.drv.chroot/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh' to '/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh': Operation not permitted
#
# There should be no reason why arion can't run without
# sandboxing, so please re-enable.
nix.useSandbox = false;
virtualisation.writableStore = true;
virtualisation.pathsInNixDB = [
# Pre-build the image because we don't want to build the world
# in the vm.
(preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml
(preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml
(preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml
pkgs.stdenv
];
virtualisation.memorySize = 512;
};
testScript = ''
$machine->fail("curl localhost:8000");
$machine->succeed("docker --version");
my $makeSubtest = sub {
my ( $subtestName, $exampleSrc ) = @_;
subtest $subtestName => sub {
$machine->succeed("rm -rf work && cp -frT $exampleSrc work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
$machine->waitUntilSucceeds("curl localhost:8000");
$machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down");
$machine->waitUntilFails("curl localhost:8000");
};
};
$makeSubtest->("minimal", "${../../examples/minimal}");
$makeSubtest->("full-nixos", "${../../examples/full-nixos}");
$makeSubtest->("nixos-unit", "${../../examples/nixos-unit}");
'';
}

59
tests/arion-test/default.nix
View file

@ -1,4 +1,4 @@
{ pkgs, ... }:
{ usePodman ? false, pkgs, lib, ... }:
let
# To make some prebuilt derivations available in the vm
@ -6,14 +6,26 @@ let
inherit modules;
inherit pkgs;
};
inherit (lib)
optionalAttrs
optionalString
;
haveSystemd = usePodman || pkgs.arionTestingFlags.dockerSupportsSystemd;
in
{
name = "arion-test";
machine = { pkgs, lib, ... }: {
environment.systemPackages = [
pkgs.arion
];
virtualisation.docker.enable = true;
] ++ lib.optional usePodman pkgs.docker;
virtualisation.docker.enable = !usePodman;
virtualisation.podman = optionalAttrs usePodman {
enable = true;
dockerSocket.enable = true;
};
# no caches, because no internet
nix.binaryCaches = lib.mkForce [];
@ -38,6 +50,7 @@ in
];
virtualisation.memorySize = 1024;
virtualisation.diskSize = 8000;
};
testScript = ''
machine.fail("curl --fail localhost:8000")
@ -57,6 +70,44 @@ in
)
machine.wait_until_fails("curl --fail localhost:8000")
# Tests
# - running same image again doesn't require a `docker load`
with subtest("docker load only once"):
# We assume image loading relies on the `docker images` and `docker load` commands, so this should fail
machine.fail(
"export REAL_DOCKER=$(which docker); rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' PATH=\"${pkgs.writeScriptBin "docker" ''
#!${pkgs.runtimeShell} -eu
echo 1>&2 "This failure is expected. Args were" "$@"
echo "$@" >/tmp/docker-args
exit 1
''}/bin:$PATH\" arion up -d"
)
machine.succeed(
"export REAL_DOCKER=$(which docker); rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' PATH=\"${pkgs.writeScriptBin "docker" ''
#!${pkgs.runtimeShell} -eu
case $1 in
load)
echo 1>&2 "arion must not docker load when upping the same deployment for the second time"
exit 1
;;
images)
echo 1>&2 "execing docker to list images"
exec $REAL_DOCKER "$@"
;;
*)
echo 1>&2 "Unknown docker invocation. This may be a shortcoming of this docker mock."
echo 1>&2 "Invocation: docker" "$@"
;;
esac
''}/bin:$PATH\" arion up -d"
)
machine.wait_until_succeeds("curl --fail localhost:8000")
machine.succeed(
"cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
)
machine.wait_until_fails("curl --fail localhost:8000")
# Tests
# - examples/flake
# This _test_ doesn't work because flake-compat fetches the github
@ -71,6 +122,7 @@ in
# machine.succeed("cd work && NIX_PATH= arion down")
# machine.wait_until_fails("curl --fail localhost:8000")
${optionalString haveSystemd ''
# Tests
# - arion exec
# - examples/full-nixos
@ -95,6 +147,7 @@ in
"cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
)
machine.wait_until_fails("curl --fail localhost:8000")
''}
# Tests
# - examples/nixos-unit

9
tests/default.nix
View file

@ -1,11 +1,16 @@
{ pkgs ? import ../pkgs.nix, nixosTestIsPerl ? false }:
{ pkgs ? import ../pkgs.nix, arionTestingFlags ? {} }:
let
inherit (pkgs) nixosTest recurseIntoAttrs arion;
in
recurseIntoAttrs {
test = if nixosTestIsPerl then nixosTest ./arion-test-perl else nixosTest ./arion-test;
test = nixosTest ./arion-test;
testWithPodman =
if arionTestingFlags.nixosHasPodmanDockerSocket
then nixosTest (pkgs.callPackage ./arion-test { usePodman = true; })
else {};
testBuild = arion.build {