Merge pull request #94 from hercules-ci/update-nixos-20.03

Update for nixos 20.03

examples/full-nixos/arion-compose.nix

@@ -1,9 +1,11 @@
 {
-  services.webserver = { pkgs, ... }: {
+  services.webserver = { pkgs, lib, ... }: {
     nixos.useSystemd = true;
     nixos.configuration.boot.tmpOnTmpfs = true;
     nixos.configuration.services.nginx.enable = true;
     nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
+    nixos.configuration.systemd.services.nginx.serviceConfig.AmbientCapabilities = 
+      lib.mkForce [ "CAP_NET_BIND_SERVICE" ];
     service.useHostStore = true;
     service.ports = [
       "8000:80" # host:container

examples/nixos-unit/arion-compose.nix

@@ -28,6 +28,9 @@
         PATH='${config.systemd.services.nginx.environment.PATH}'
         echo nginx:x:${toString config.users.users.nginx.uid}:${toString config.users.groups.nginx.gid}:nginx web server user:/var/empty:/bin/sh >>/etc/passwd
         echo nginx:x:${toString config.users.groups.nginx.gid}:nginx >>/etc/group
+        echo 'nobody:x:65534:65534:Unprivileged account do not use:/var/empty:/run/current-system/sw/bin/nologin' >>/etc/passwd
+        echo 'nogroup:x:65534:' >>/etc/group
+        mkdir -p /run/nginx/ /var/spool/nginx/logs/
         ${config.systemd.services.nginx.runner}
       '';
     };

nix/ci.nix

@@ -1,25 +1,35 @@
 let
   sources = import ./sources.nix;
   lib = import (sources."nixpkgs" + "/lib");
-  inherit (import sources."project.nix" { inherit lib; }) dimension;
+  inherit (import (sources."project.nix" + "/lib/dimension.nix") { inherit lib; }) dimension;
 in
 
 dimension "Nixpkgs version" {
     "nixos-19_03" = {
-      nixpkgsSource = "nixpkgs";
+      # flyingcircus.io latest long-term support is based off 19.03
-      isReferenceNixpkgs = true;
+      # https://flyingcircus.io/doc/
+      # It is nice to have some level of support for their platform,
+      # but we don't guarantee any support.
+      nixpkgsSource = "nixos-19.03";
       enableDoc = false;
+      nixosTestIsPerl = true;
     };
     "nixos-19_09" = {
       nixpkgsSource = "nixos-19.09";
+      enableDoc = false;
+      nixosTestIsPerl = true;
+    };
+    "nixos-20_03" = {
+      nixpkgsSource = "nixos-20.03";
+      isReferenceNixpkgs = true;
       enableDoc = true;
     };
-    "nixos-unstable" = {
+    # "nixos-unstable" = {
-      nixpkgsSource = "nixos-unstable";
+    #   nixpkgsSource = "nixos-unstable";
-      enableDoc = true;
+    #   enableDoc = true;
-    };
+    # };
   } (
-    _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true }:
+    _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true, nixosTestIsPerl ? false }:
 
 
       dimension "System" {
@@ -28,7 +38,10 @@ dimension "Nixpkgs version" {
       } (
         system: { isReferenceTarget ? false, enableNixOSTests ? true }:
           let
-            pkgs = import ./. { inherit system; nixpkgsSrc = sources.${nixpkgsSource}; };
+            pkgs = import ./. {
+              inherit system nixosTestIsPerl;
+              nixpkgsSrc = sources.${nixpkgsSource};
+            };
           in
           {
             inherit (pkgs) arion;

nix/default.nix

@@ -1,7 +1,8 @@
 { sources ? import ./sources.nix
-, nixpkgsName ? "nixos-19.09"
+, nixpkgsName ? "nixos-20.03"
 , nixpkgsSrc ? sources.${nixpkgsName}
 , system ? builtins.currentSystem
+, nixosTestIsPerl ? false
 , ...
 }:
 
@@ -11,6 +12,7 @@ import nixpkgsSrc ({
   };
   overlays = [
     # all the packages are defined there:
+    (_: _: { inherit nixosTestIsPerl; })
     (import ./overlay.nix)
   ];
   inherit system;

nix/sources.json

@@ -5,10 +5,22 @@
         "homepage": "https://github.com/nmattia/niv",
         "owner": "nmattia",
         "repo": "niv",
-        "rev": "1dd094156b249586b66c16200ecfd365c7428dc0",
+        "rev": "98c74a80934123cb4c3bf3314567f67311eb711a",
-        "sha256": "1b2vjnn8iac5iiqszjc2v1s1ygh0yri998c0k3s4x4kn0dsqik21",
+        "sha256": "1w8n54hapd4x9f1am33icvngkqns7m3hl9yair38yqq08ffwg0kn",
         "type": "tarball",
-        "url": "https://github.com/nmattia/niv/archive/1dd094156b249586b66c16200ecfd365c7428dc0.tar.gz",
+        "url": "https://github.com/nmattia/niv/archive/98c74a80934123cb4c3bf3314567f67311eb711a.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixos-19.03": {
+        "branch": "nixos-19.03",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs-channels",
+        "rev": "34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59",
+        "sha256": "11z6ajj108fy2q5g8y4higlcaqncrbjm3dnv17pvif6avagw4mcb",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/34c7eb7545d155cc5b6f499b23a7cb1c96ab4d59.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-19.09": {
@@ -23,29 +35,41 @@
         "url": "https://github.com/NixOS/nixpkgs-channels/archive/3ba0d9f75ccffd41e32cfea4046805f8bbab12f5.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
+    "nixos-20.03": {
+        "branch": "release-20.03",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d6e406ddaea2e690c2f9f1a283e44c3d1c588ba3",
+        "sha256": "1m8gyrp8cpmnmxv3g2pv1460nz10bb88zqzvj6wmnhqkjgwwj3hm",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/d6e406ddaea2e690c2f9f1a283e44c3d1c588ba3.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "nixos-unstable": {
         "branch": "nixos-unstable",
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
         "repo": "nixpkgs-channels",
-        "rev": "cb4332e3eb6dfdb653f1fc7397a0292df228a533",
+        "rev": "a2e06fc3423c4be53181b15c28dfbe0bcf67dd73",
-        "sha256": "1722wphznqhpfny08rcy19l85r2l893ckjc3h1vfivj6aj64fwjr",
+        "sha256": "0bjx4iq6nyhj47q5zkqsbfgng445xwprrslj1xrv56142jn8n5r9",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/cb4332e3eb6dfdb653f1fc7397a0292df228a533.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/a2e06fc3423c4be53181b15c28dfbe0bcf67dd73.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
         "version": ""
     },
     "nixpkgs": {
-        "branch": "nixos-19.03",
+        "branch": "nixos-20.03",
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
         "repo": "nixpkgs-channels",
-        "rev": "d15a31f88a261281cd7c79038ae860c5ed95507d",
+        "rev": "99a3d7a86fce9e9c9f23b3e304d7d2b1270a12b8",
-        "sha256": "038iqfwmppnxq6aa89qm6k98lhwg686bmc9qjifibddm8pcp2wd0",
+        "sha256": "0i40cl3n6600z2lkwrpiy28dcnv2r63fcgfswj91aaf1xfn2chql",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/d15a31f88a261281cd7c79038ae860c5ed95507d.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/99a3d7a86fce9e9c9f23b3e304d7d2b1270a12b8.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "project.nix": {
@@ -54,10 +78,10 @@
         "homepage": null,
         "owner": "hercules-ci",
         "repo": "project.nix",
-        "rev": "33e5f3cb25feff4ccd00f8c60a05976e2ee01802",
+        "rev": "95f26b1cca0414f080172721ab7996ab65b8d968",
-        "sha256": "0c3q3il5h6q3ms8m6da51knvjsfvpz12sh3a3av4d2a5ikm5ncl1",
+        "sha256": "1fbpvidq3bdvinxdkk2vvvfjhxmbf45hgy2sgv2pbml4zrckbfdj",
         "type": "tarball",
-        "url": "https://github.com/hercules-ci/project.nix/archive/33e5f3cb25feff4ccd00f8c60a05976e2ee01802.tar.gz",
+        "url": "https://github.com/hercules-ci/project.nix/archive/95f26b1cca0414f080172721ab7996ab65b8d968.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "releaser": {
@@ -66,10 +90,10 @@
         "homepage": null,
         "owner": "domenkozar",
         "repo": "releaser",
-        "rev": "43a4e27654f388e8eacab631e24e26792ff88fe2",
+        "rev": "0be40041273bd93891dd2be300d1f21f9e9a121b",
-        "sha256": "072jlbw0hdc4nvs9frd7wdyzdv4mz2dc5ib35iaqi9rzdafq6822",
+        "sha256": "0ckgcliyi37hvpfp40nmk6r0q5irinkc2cpqs0l85z2a7si66hzh",
         "type": "tarball",
-        "url": "https://github.com/domenkozar/releaser/archive/43a4e27654f388e8eacab631e24e26792ff88fe2.tar.gz",
+        "url": "https://github.com/domenkozar/releaser/archive/0be40041273bd93891dd2be300d1f21f9e9a121b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

src/nix/modules/composition/arion-base-image.nix

@@ -10,7 +10,7 @@ let
   tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath));
   name = "arion-base";
 
-  builtImage = pkgs.dockerTools.buildLayeredImage {
+  builtImage = pkgs.dockerTools.buildImage {
     inherit name;
     contents = pkgs.runCommand "minimal-contents" {} ''
         mkdir -p $out/bin $out/usr/bin

tests/arion-test-perl/README.md

@@ -0,0 +1,4 @@
+
+This test suite exists only to keep tests around for older versions of NixOS.
+
+This will be removed when 19.09 becomes irrelevant.

tests/arion-test-perl/default.nix

@@ -0,0 +1,60 @@
+{ pkgs, ... }:
+
+let
+  # To make some prebuilt derivations available in the vm
+  preEval = modules: import ../../src/nix/eval-composition.nix {
+    inherit modules;
+    inherit pkgs;
+  };
+in
+{
+  name = "arion-test";
+  machine = { pkgs, lib, ... }: {
+    environment.systemPackages = [
+      pkgs.arion
+    ];
+    virtualisation.docker.enable = true;
+    
+    # no caches, because no internet
+    nix.binaryCaches = lib.mkForce [];
+
+    # FIXME: Sandbox seems broken with current version of NixOS test
+    #        w/ writable store. Error:
+    #        machine# error: linking '/nix/store/7r8z2zvhwda85pgpdn5hzzz6hs1njklc-stdenv-linux.drv.chroot/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh' to '/nix/store/6v3y7s4q4wd16hsw393gjpxvcf9159bv-patch-shebangs.sh': Operation not permitted
+    #
+    #        There should be no reason why arion can't run without
+    #        sandboxing, so please re-enable.
+    nix.useSandbox = false;
+
+    virtualisation.writableStore = true;
+    virtualisation.pathsInNixDB = [
+      # Pre-build the image because we don't want to build the world
+      # in the vm.
+      (preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml
+      (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml
+      (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml
+      pkgs.stdenv
+    ];
+
+    virtualisation.memorySize = 512;
+  };
+  testScript = ''
+    $machine->fail("curl localhost:8000");
+    $machine->succeed("docker --version");
+
+    my $makeSubtest = sub {
+      my ( $subtestName, $exampleSrc ) = @_;
+
+      subtest $subtestName => sub {
+        $machine->succeed("rm -rf work && cp -frT $exampleSrc work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
+        $machine->waitUntilSucceeds("curl localhost:8000");
+        $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down");
+        $machine->waitUntilFails("curl localhost:8000");
+      };
+    };
+
+    $makeSubtest->("minimal", "${../../examples/minimal}");
+    $makeSubtest->("full-nixos", "${../../examples/full-nixos}");
+    $makeSubtest->("nixos-unit", "${../../examples/nixos-unit}");
+  '';
+}

tests/arion-test/default.nix

@@ -35,32 +35,62 @@ in
       (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml
       pkgs.stdenv
     ];
+
+    virtualisation.memorySize = 512;
   };
   testScript = ''
-    $machine->fail("curl localhost:8000");
+    machine.fail("curl localhost:8000")
-    $machine->succeed("docker --version");
+    machine.succeed("docker --version")
 
-    subtest "minimal", sub {
+    # Tests
-      $machine->succeed("cp -r ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
+    #  - arion up
-      $machine->waitUntilSucceeds("curl localhost:8000");
+    #  - arion down
-      $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down && rm -rf work");
+    #  - examples/minimal
-      $machine->waitUntilFails("curl localhost:8000");
+    with subtest("minimal"):
-    };
+        machine.succeed(
+            "rm -rf work && cp -frT ${../../examples/minimal} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"
+        )
+        machine.wait_until_succeeds("curl localhost:8000")
+        machine.succeed(
+            "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
+        )
+        machine.wait_until_fails("curl localhost:8000")
 
-    subtest "full-nixos", sub {
+    # Tests
-      $machine->succeed("cp -r ${../../examples/full-nixos} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
+    #  - arion exec
-      $machine->waitUntilSucceeds("curl localhost:8000");
+    #  - examples/full-nixos
-      # Also test exec with defaultExec
+    with subtest("full-nixos"):
-      $machine->succeed("cd work && export NIX_PATH=nixpkgs='${pkgs.path}' && (echo 'nix run -f ~/h/arion arion -c arion exec webserver'; echo 'target=world; echo Hello \$target'; echo exit) | script /dev/null | grep 'Hello world'");
+        machine.succeed(
-      $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down && rm -rf work");
+            "rm -rf work && cp -frT ${../../examples/full-nixos} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"
-      $machine->waitUntilFails("curl localhost:8000");
+        )
-    };
+        machine.wait_until_succeeds("curl localhost:8000")
 
-    subtest "nixos-unit", sub {
+        machine.succeed(
-      $machine->succeed("cp -r ${../../examples/nixos-unit} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d");
+            """
-      $machine->waitUntilSucceeds("curl localhost:8000");
+            set -eux -o pipefail
-      $machine->succeed("cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down && rm -rf work");
+            cd work
-      $machine->waitUntilFails("curl localhost:8000");
+            export NIX_PATH=nixpkgs='${pkgs.path}'
-    };
+            echo 'target=world; echo Hello $target; exit' \
+              | script 'arion exec webserver' \
+              | grep 'Hello world'
+            """
+        ),
+
+        machine.succeed(
+            "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
+        )
+        machine.wait_until_fails("curl localhost:8000")
+
+    # Tests
+    #  - examples/nixos-unit
+    with subtest("nixos-unit"):
+        machine.succeed(
+            "rm -rf work && cp -frT ${../../examples/nixos-unit} work && cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion up -d"
+        )
+        machine.wait_until_succeeds("curl localhost:8000")
+        machine.succeed(
+            "cd work && NIX_PATH=nixpkgs='${pkgs.path}' arion down"
+        )
+        machine.wait_until_fails("curl localhost:8000")
   '';
 }

tests/default.nix

@@ -1,11 +1,11 @@
-{ pkgs ? import ../pkgs.nix }:
+{ pkgs ? import ../pkgs.nix, nixosTestIsPerl ? false }:
 let
   inherit (pkgs) nixosTest recurseIntoAttrs arion;
 in
 
 recurseIntoAttrs {
 
-  test = nixosTest ./arion-test;
+  test = if nixosTestIsPerl then nixosTest ./arion-test-perl else nixosTest ./arion-test;
 
   testBuild = arion.build {