From 35a309097a68aadccfa3e92b5f0e3b5cc86559c4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 Mar 2019 19:41:54 +0100 Subject: [PATCH 1/4] Add support for NixOS inside containers --- examples/full-nixos/arion-compose.nix | 11 ++++ examples/full-nixos/arion-pkgs.nix | 2 + examples/nixos-unit/arion-compose.nix | 40 +++++++++++++ examples/nixos-unit/arion-pkgs.nix | 2 + src/nix/eval-service.nix | 2 + src/nix/modules/nixos/container-systemd.nix | 24 ++++++++ .../service/docker-compose-service.nix | 23 ++++++- src/nix/modules/service/nixos-init.nix | 39 ++++++++++++ src/nix/modules/service/nixos.nix | 60 +++++++++++++++++++ tests/arion-test/default.nix | 31 ++++++++-- 10 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 examples/full-nixos/arion-compose.nix create mode 100644 examples/full-nixos/arion-pkgs.nix create mode 100644 examples/nixos-unit/arion-compose.nix create mode 100644 examples/nixos-unit/arion-pkgs.nix create mode 100644 src/nix/modules/nixos/container-systemd.nix create mode 100644 src/nix/modules/service/nixos-init.nix create mode 100644 src/nix/modules/service/nixos.nix diff --git a/examples/full-nixos/arion-compose.nix b/examples/full-nixos/arion-compose.nix new file mode 100644 index 0000000..02c9973 --- /dev/null +++ b/examples/full-nixos/arion-compose.nix @@ -0,0 +1,11 @@ +{ + docker-compose.services.webserver = { pkgs, ... }: { + nixos.useInit = true; + nixos.configuration.services.nginx.enable = true; + nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual"; + service.useHostStore = true; + service.ports = [ + "8000:80" # host:container + ]; + }; +} diff --git a/examples/full-nixos/arion-pkgs.nix b/examples/full-nixos/arion-pkgs.nix new file mode 100644 index 0000000..d13a605 --- /dev/null +++ b/examples/full-nixos/arion-pkgs.nix @@ -0,0 +1,2 @@ +# Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH +import {} diff --git a/examples/nixos-unit/arion-compose.nix b/examples/nixos-unit/arion-compose.nix new file mode 100644 index 0000000..80105b1 --- /dev/null +++ b/examples/nixos-unit/arion-compose.nix @@ -0,0 +1,40 @@ + +/* + + DISCLAIMER + + This uses a somewhat hidden feature in NixOS, which is the + "runner". It's a script that's available on systemd services + that lets you run the service independently from systemd. + However, it was clearly not intended for public consumption + so please use it with care. + It does not support all features of systemd so you are on + your own if you use it in production. + + One known issue is that the script does not respond to docker's + SIGTERM shutdown signal. + + */ + +{ + docker-compose.services.webserver = { config, pkgs, ... }: { + + nixos.configuration = {config, pkgs, ...}: { + boot.isContainer = true; + services.nginx.enable = true; + services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual"; + system.build.run-nginx = pkgs.writeScript "run-nginx" '' + #!${pkgs.bash}/bin/bash + PATH='${config.systemd.services.nginx.environment.PATH}' + echo nginx:x:${toString config.users.users.nginx.uid}:${toString config.users.groups.nginx.gid}:nginx web server user:/var/empty:/bin/sh >>/etc/passwd + echo nginx:x:${toString config.users.groups.nginx.gid}:nginx >>/etc/group + ${config.systemd.services.nginx.runner} + ''; + }; + service.command = [ config.nixos.build.run-nginx ]; + service.useHostStore = true; + service.ports = [ + "8000:80" # host:container + ]; + }; +} diff --git a/examples/nixos-unit/arion-pkgs.nix b/examples/nixos-unit/arion-pkgs.nix new file mode 100644 index 0000000..d13a605 --- /dev/null +++ b/examples/nixos-unit/arion-pkgs.nix @@ -0,0 +1,2 @@ +# Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH +import {} diff --git a/src/nix/eval-service.nix b/src/nix/eval-service.nix index cf31678..f2273d1 100644 --- a/src/nix/eval-service.nix +++ b/src/nix/eval-service.nix @@ -12,6 +12,8 @@ let ./modules/service/docker-compose-service.nix ./modules/service/host-store.nix ./modules/service/host.nix + ./modules/service/nixos.nix + ./modules/service/nixos-init.nix ]; argsModule = { diff --git a/src/nix/modules/nixos/container-systemd.nix b/src/nix/modules/nixos/container-systemd.nix new file mode 100644 index 0000000..d97812f --- /dev/null +++ b/src/nix/modules/nixos/container-systemd.nix @@ -0,0 +1,24 @@ +/* + NixOS configuration to for running a mostly normal systemd-based + NixOS in Docker. + */ +{ pkgs, lib, ...}: { + + # imports = [ + # # This profile doesn't seem to work well. + # (pkgs.path + "/nixos/modules/profiles/docker-container.nix") + # This one works, but can not be imported here due because imports can not depend on pkgs. + # (pkgs.path + "/nixos/modules/profiles/minimal.nix") + # ]; + boot.isContainer = true; + boot.specialFileSystems = lib.mkForce {}; + networking.hostName = ""; + + services.journald.console = "/dev/console"; + + systemd.services.systemd-logind.enable = false; + systemd.services.console-getty.enable = false; + + systemd.sockets.nix-daemon.enable = false; + systemd.services.nix-daemon.enable = false; +} diff --git a/src/nix/modules/service/docker-compose-service.nix b/src/nix/modules/service/docker-compose-service.nix index 82963a8..b708e4a 100644 --- a/src/nix/modules/service/docker-compose-service.nix +++ b/src/nix/modules/service/docker-compose-service.nix @@ -36,7 +36,12 @@ in service.volumes = mkOption { type = listOf types.unspecified; default = []; - description = ""; + description = dockerComposeRef "volumes"; + }; + service.tmpfs = mkOption { + type = listOf types.str; + default = []; + description = dockerComposeRef "tmpfs"; }; service.build.context = mkOption { type = nullOr str; @@ -52,6 +57,11 @@ in default = null; description = dockerComposeKitchenSink; }; + service.tty = mkOption { + type = nullOr bool; + default = null; + description = dockerComposeKitchenSink; + }; service.environment = mkOption { type = attrsOf (either str int); default = {}; @@ -130,6 +140,11 @@ in default = null; description = dockerComposeRef "network_mode"; }; + service.stop_signal = mkOption { + type = nullOr str; + default = null; + description = dockerComposeRef "stop_signal"; + }; }; config.build.service = { @@ -166,6 +181,12 @@ in inherit (config.service) network_mode; } // lib.optionalAttrs (config.service.restart != null) { inherit (config.service) restart; + } // lib.optionalAttrs (config.service.stop_signal != null) { + inherit (config.service) stop_signal; + } // lib.optionalAttrs (config.service.tmpfs != []) { + inherit (config.service) tmpfs; + } // lib.optionalAttrs (config.service.tty != null) { + inherit (config.service) tty; } // lib.optionalAttrs (config.service.working_dir != null) { inherit (config.service) working_dir; }; diff --git a/src/nix/modules/service/nixos-init.nix b/src/nix/modules/service/nixos-init.nix new file mode 100644 index 0000000..39ae4f6 --- /dev/null +++ b/src/nix/modules/service/nixos-init.nix @@ -0,0 +1,39 @@ +/* + Invokes the NixOS init system in the container. + */ +{ config, lib, pkgs, ... }: +let + inherit (lib) types; +in +{ + options = { + nixos.useInit = lib.mkOption { + type = types.bool; + default = false; + description = '' + When enabled, call the NixOS init system. + + Configure NixOS with nixos.configuration. + ''; + }; + }; + + config = lib.mkIf (config.nixos.useInit) { + nixos.configuration.imports = [ + ../nixos/container-systemd.nix + (pkgs.path + "/nixos/modules/profiles/minimal.nix") + ]; + service.command = [ "${config.nixos.build.toplevel}/init" ]; + service.environment.container = "docker"; + service.volumes = [ + "/sys/fs/cgroup:/sys/fs/cgroup:ro" + ]; + service.tmpfs = [ + "/tmp" + "/run" + "/run/wrappers" + ]; + service.stop_signal = "SIGRTMIN+3"; + service.tty = true; + }; +} diff --git a/src/nix/modules/service/nixos.nix b/src/nix/modules/service/nixos.nix new file mode 100644 index 0000000..fc18898 --- /dev/null +++ b/src/nix/modules/service/nixos.nix @@ -0,0 +1,60 @@ +/* + Generic options for adding NixOS modules and evaluating the NixOS + configuration. The methods for installing NixOS in the image are + defined elsewhere. + */ +{ pkgs, lib, config, ... }: +let + inherit (lib) types; + inherit (types) attrs; +in +{ + options = { + nixos.configuration = lib.mkOption { + type = with types; coercedTo unspecified (a: [a]) (listOf unspecified); + default = {}; + description = '' + Modules to add to the NixOS configuration. + + This option is unused by default, because not all images use NixOS. + + One way to use this is to enable nixos.useInit, but the + NixOS configuration can be used in other ways. + ''; + }; + + nixos.build = lib.mkOption { + type = attrs; + readOnly = true; + description = '' + NixOS build products from config.system.build, such as toplevel and etc. + + This option is unused by default, because not all images use NixOS. + + One way to use this is to enable nixos.useInit, but the + NixOS configuration can be used in other ways. + ''; + }; + + nixos.evaluatedConfig = lib.mkOption { + type = attrs; + readOnly = true; + description = '' + Evaluated NixOS configuration, to be read by service-level modules. + + This option is unused by default, because not all images use NixOS. + + One way to use this is to enable nixos.useInit, but the + NixOS configuration can be used in other ways. + ''; + }; + }; + + config = { + nixos.build = builtins.addErrorContext "while evaluating the service nixos configuration" ( + pkgs.nixos config.nixos.configuration + ); + nixos.configuration = { config, ... }: { system.build.theConfig = config; }; + nixos.evaluatedConfig = config.nixos.build.theConfig; + }; +} diff --git a/tests/arion-test/default.nix b/tests/arion-test/default.nix index e301b10..4ad5aca 100644 --- a/tests/arion-test/default.nix +++ b/tests/arion-test/default.nix @@ -2,8 +2,8 @@ let # To make some prebuilt derivations available in the vm - preEval = import ../../src/nix/eval-composition.nix { - modules = [ ../../examples/minimal/arion-compose.nix ]; + preEval = modules: import ../../src/nix/eval-composition.nix { + inherit modules; inherit pkgs; }; in @@ -31,14 +31,35 @@ in virtualisation.pathsInNixDB = [ # Pre-build the image because we don't want to build the world # in the vm. - preEval.config.build.dockerComposeYaml + (preEval [ ../../examples/minimal/arion-compose.nix ]).config.build.dockerComposeYaml + (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.build.dockerComposeYaml + (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.build.dockerComposeYaml pkgs.stdenv ]; }; testScript = '' $machine->fail("curl localhost:8000"); $machine->succeed("docker --version"); - $machine->succeed("cp -r ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"); - $machine->waitUntilSucceeds("curl localhost:8000"); + + subtest "minimal", sub { + $machine->succeed("cp -r ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"); + $machine->waitUntilSucceeds("curl localhost:8000"); + $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down && rm -rf work"); + $machine->waitUntilFails("curl localhost:8000"); + }; + + subtest "full-nixos", sub { + $machine->succeed("cp -r ${../../examples/full-nixos} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"); + $machine->waitUntilSucceeds("curl localhost:8000"); + $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down && rm -rf work"); + $machine->waitUntilFails("curl localhost:8000"); + }; + + subtest "nixos-unit", sub { + $machine->succeed("cp -r ${../../examples/nixos-unit} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"); + $machine->waitUntilSucceeds("curl localhost:8000"); + $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down && rm -rf work"); + $machine->waitUntilFails("curl localhost:8000"); + }; ''; } From c5e6fe6e5f5fe3a2c9660e3823e29bf11b621e06 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 Mar 2019 19:42:10 +0100 Subject: [PATCH 2/4] Remove redundant buildInput --- doc/manual/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/default.nix b/doc/manual/default.nix index 0e7fcba..59f62ab 100644 --- a/doc/manual/default.nix +++ b/doc/manual/default.nix @@ -15,7 +15,7 @@ let optionsXML = # builtins.toFile "options.xml" (builtins.toXML optionsList); pkgs.runCommand "options.xml" { - buildInputs = [pkgs.nix pkgs.fakeroot pkgs.jq]; + buildInputs = [pkgs.nix pkgs.jq]; inherit optionsExpr; } '' export NIX_LOG_DIR=$PWD From a5e08b23a34f428dec76b86de1e904b3d6e7e3f0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 Mar 2019 19:42:32 +0100 Subject: [PATCH 3/4] Add service.useHostNixDaemon --- src/nix/modules/service/host-store.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nix/modules/service/host-store.nix b/src/nix/modules/service/host-store.nix index ae6a1e5..2060517 100644 --- a/src/nix/modules/service/host-store.nix +++ b/src/nix/modules/service/host-store.nix @@ -16,6 +16,11 @@ in default = false; description = "Bind mounts the host store if enabled, avoiding copying."; }; + service.useHostNixDaemon = mkOption { + type = types.bool; + default = false; + description = "Make the host Nix daemon available."; + }; }; config = mkIf config.service.useHostStore { service.image = "arion-base"; @@ -23,6 +28,6 @@ in service.volumes = [ "${config.host.nixStorePrefix}/nix/store:/nix/store" "${config.host.nixStorePrefix}${pkgs.buildEnv { name = "container-system-env"; paths = [ pkgs.bashInteractive pkgs.coreutils ]; }}:/run/system" - ]; + ] ++ lib.optional config.service.useHostNixDaemon "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket"; }; } From 97d58f7bd3bbf293bc6ff55ed3ba6d1c8181e23f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Mar 2019 12:49:39 +0100 Subject: [PATCH 4/4] Rename nixos.useInit -> nixos.useSystemd --- examples/full-nixos/arion-compose.nix | 2 +- src/nix/modules/service/nixos-init.nix | 6 +++--- src/nix/modules/service/nixos.nix | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/full-nixos/arion-compose.nix b/examples/full-nixos/arion-compose.nix index 02c9973..03eb78f 100644 --- a/examples/full-nixos/arion-compose.nix +++ b/examples/full-nixos/arion-compose.nix @@ -1,6 +1,6 @@ { docker-compose.services.webserver = { pkgs, ... }: { - nixos.useInit = true; + nixos.useSystemd = true; nixos.configuration.services.nginx.enable = true; nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual"; service.useHostStore = true; diff --git a/src/nix/modules/service/nixos-init.nix b/src/nix/modules/service/nixos-init.nix index 39ae4f6..2b37db8 100644 --- a/src/nix/modules/service/nixos-init.nix +++ b/src/nix/modules/service/nixos-init.nix @@ -7,18 +7,18 @@ let in { options = { - nixos.useInit = lib.mkOption { + nixos.useSystemd = lib.mkOption { type = types.bool; default = false; description = '' - When enabled, call the NixOS init system. + When enabled, call the NixOS systemd-based init system. Configure NixOS with nixos.configuration. ''; }; }; - config = lib.mkIf (config.nixos.useInit) { + config = lib.mkIf (config.nixos.useSystemd) { nixos.configuration.imports = [ ../nixos/container-systemd.nix (pkgs.path + "/nixos/modules/profiles/minimal.nix") diff --git a/src/nix/modules/service/nixos.nix b/src/nix/modules/service/nixos.nix index fc18898..2cfc344 100644 --- a/src/nix/modules/service/nixos.nix +++ b/src/nix/modules/service/nixos.nix @@ -18,7 +18,7 @@ in This option is unused by default, because not all images use NixOS. - One way to use this is to enable nixos.useInit, but the + One way to use this is to enable nixos.useSystemd, but the NixOS configuration can be used in other ways. ''; }; @@ -31,7 +31,7 @@ in This option is unused by default, because not all images use NixOS. - One way to use this is to enable nixos.useInit, but the + One way to use this is to enable nixos.useSystemd, but the NixOS configuration can be used in other ways. ''; }; @@ -44,7 +44,7 @@ in This option is unused by default, because not all images use NixOS. - One way to use this is to enable nixos.useInit, but the + One way to use this is to enable nixos.useSystemd, but the NixOS configuration can be used in other ways. ''; };