Merge pull request #99 from hercules-ci/update

Update

examples/nixos-unit/arion-compose.nix

@@ -19,10 +19,15 @@
 {
   services.webserver = { config, pkgs, ... }: {
 
-    nixos.configuration = {config, pkgs, ...}: {
+    nixos.configuration = {config, lib, options, pkgs, ...}: {
       boot.isContainer = true;
-      services.nginx.enable = true;
+      services.nginx = {
-      services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
+        enable = true;
+        virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
+      } // lib.optionalAttrs (options?services.nginx.stateDir) {
+        # Work around a problem in NixOS 20.03
+        stateDir = "/var/lib/nginx";
+      };
       system.build.run-nginx = pkgs.writeScript "run-nginx" ''
         #!${pkgs.bash}/bin/bash
         PATH='${config.systemd.services.nginx.environment.PATH}'
@@ -30,7 +35,8 @@
         echo nginx:x:${toString config.users.groups.nginx.gid}:nginx >>/etc/group
         echo 'nobody:x:65534:65534:Unprivileged account do not use:/var/empty:/run/current-system/sw/bin/nologin' >>/etc/passwd
         echo 'nogroup:x:65534:' >>/etc/group
-        mkdir -p /run/nginx/ /var/spool/nginx/logs/
+        mkdir -p /var/log/nginx /run/nginx/ /var/cache/nginx /var/lib/nginx/{,logs,proxy_temp,client_body_temp,fastcgi_temp,scgi_temp,uwsgi_temp}
+        chown nginx /var/log/nginx /run/nginx/ /var/cache/nginx /var/lib/nginx/{,logs,proxy_temp,client_body_temp,fastcgi_temp,scgi_temp,uwsgi_temp}
         ${config.systemd.services.nginx.runner}
       '';
     };

nix/ci.nix

@@ -1,6 +1,6 @@
 let
   sources = import ./sources.nix;
-  lib = import (sources."nixpkgs" + "/lib");
+  lib = import (sources."nixos-20.03" + "/lib");
   inherit (import (sources."project.nix" + "/lib/dimension.nix") { inherit lib; }) dimension;
 in
 
@@ -21,13 +21,18 @@ dimension "Nixpkgs version" {
     };
     "nixos-20_03" = {
       nixpkgsSource = "nixos-20.03";
+      isReferenceNixpkgs = false;
+      enableDoc = true;
+    };
+    "nixos-20_09" = {
+      nixpkgsSource = "nixos-20.09";
       isReferenceNixpkgs = true;
       enableDoc = true;
     };
-    # "nixos-unstable" = {
+    "nixos-unstable" = {
-    #   nixpkgsSource = "nixos-unstable";
+      nixpkgsSource = "nixos-unstable";
-    #   enableDoc = true;
+      enableDoc = true;
-    # };
+    };
   } (
     _name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true, nixosTestIsPerl ? false }:
 

nix/default.nix

@@ -1,5 +1,5 @@
 { sources ? import ./sources.nix
-, nixpkgsName ? "nixos-20.03"
+, nixpkgsName ? "nixos-unstable"
 , nixpkgsSrc ? sources.${nixpkgsName}
 , system ? builtins.currentSystem
 , nixosTestIsPerl ? false

nix/overlay.nix

@@ -48,7 +48,7 @@ in
         haskellPkgs.ghcid
         super.docker-compose
         self.niv
-        self.releaser
+        # self.releaser
       ];
     };
   };

nix/sources.json

@@ -5,10 +5,10 @@
         "homepage": "https://github.com/nmattia/niv",
         "owner": "nmattia",
         "repo": "niv",
-        "rev": "98c74a80934123cb4c3bf3314567f67311eb711a",
+        "rev": "fad2a6cbfb2e7cdebb7cb0ad2f5cc91e2c9bc06b",
-        "sha256": "1w8n54hapd4x9f1am33icvngkqns7m3hl9yair38yqq08ffwg0kn",
+        "sha256": "0mghc1j0rd15spdjx81bayjqr0khc062cs25y5dcfzlxk4ynyc6m",
         "type": "tarball",
-        "url": "https://github.com/nmattia/niv/archive/98c74a80934123cb4c3bf3314567f67311eb711a.tar.gz",
+        "url": "https://github.com/nmattia/niv/archive/fad2a6cbfb2e7cdebb7cb0ad2f5cc91e2c9bc06b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-19.03": {
@@ -29,10 +29,10 @@
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
         "repo": "nixpkgs-channels",
-        "rev": "3ba0d9f75ccffd41e32cfea4046805f8bbab12f5",
+        "rev": "289466dd6a11c65a7de4a954d6ebf66c1ad07652",
-        "sha256": "0w20drs4mwlq12k1sss1x8adyf5ph5jd52n8wdcgmn4sm60qjmki",
+        "sha256": "0r5ja052s86fr54fm1zlhld3fwawz2w1d1gd6vbvpjrpjfyajibn",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/3ba0d9f75ccffd41e32cfea4046805f8bbab12f5.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/289466dd6a11c65a7de4a954d6ebf66c1ad07652.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-20.03": {
@@ -41,10 +41,22 @@
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d6e406ddaea2e690c2f9f1a283e44c3d1c588ba3",
+        "rev": "b4db68ff563895eea6aab4ff24fa04ef403dfe14",
-        "sha256": "1m8gyrp8cpmnmxv3g2pv1460nz10bb88zqzvj6wmnhqkjgwwj3hm",
+        "sha256": "1qbs7p0mmcmpg70ibd437hl57byqx5q0pc61p1dckrkazj7kq0pc",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/d6e406ddaea2e690c2f9f1a283e44c3d1c588ba3.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/b4db68ff563895eea6aab4ff24fa04ef403dfe14.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixos-20.09": {
+        "branch": "nixos-20.09",
+        "description": "Nix Packages collection",
+        "homepage": null,
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "0cfe5377e8993052f9b0dd56d058f8008af45bd9",
+        "sha256": "0i3ybddi2mrlaz3di3svdpgy93zwmdglpywih4s9rd3wj865gzn1",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/0cfe5377e8993052f9b0dd56d058f8008af45bd9.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixos-unstable": {
@@ -52,36 +64,24 @@
         "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
         "homepage": "https://github.com/NixOS/nixpkgs",
         "owner": "NixOS",
-        "repo": "nixpkgs-channels",
+        "repo": "nixpkgs",
-        "rev": "a2e06fc3423c4be53181b15c28dfbe0bcf67dd73",
+        "rev": "c59ea8b8a0e7f927e7291c14ea6cd1bd3a16ff38",
-        "sha256": "0bjx4iq6nyhj47q5zkqsbfgng445xwprrslj1xrv56142jn8n5r9",
+        "sha256": "1ak7jqx94fjhc68xh1lh35kh3w3ndbadprrb762qgvcfb8351x8v",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/a2e06fc3423c4be53181b15c28dfbe0bcf67dd73.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/c59ea8b8a0e7f927e7291c14ea6cd1bd3a16ff38.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
         "version": ""
     },
-    "nixpkgs": {
-        "branch": "nixos-20.03",
-        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
-        "homepage": "https://github.com/NixOS/nixpkgs",
-        "owner": "NixOS",
-        "repo": "nixpkgs-channels",
-        "rev": "99a3d7a86fce9e9c9f23b3e304d7d2b1270a12b8",
-        "sha256": "0i40cl3n6600z2lkwrpiy28dcnv2r63fcgfswj91aaf1xfn2chql",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs-channels/archive/99a3d7a86fce9e9c9f23b3e304d7d2b1270a12b8.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "project.nix": {
         "branch": "master",
         "description": "A configuration manager for your projects",
         "homepage": null,
         "owner": "hercules-ci",
         "repo": "project.nix",
-        "rev": "95f26b1cca0414f080172721ab7996ab65b8d968",
+        "rev": "2e598501e7fda6993d2a1a281aa296b26d01e10c",
-        "sha256": "1fbpvidq3bdvinxdkk2vvvfjhxmbf45hgy2sgv2pbml4zrckbfdj",
+        "sha256": "1rkzpzxpg69px6qwchdlg4xf5irv0snrzk2l6vrs9rsx48gqax9j",
         "type": "tarball",
-        "url": "https://github.com/hercules-ci/project.nix/archive/95f26b1cca0414f080172721ab7996ab65b8d968.tar.gz",
+        "url": "https://github.com/hercules-ci/project.nix/archive/2e598501e7fda6993d2a1a281aa296b26d01e10c.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "releaser": {
@@ -90,10 +90,10 @@
         "homepage": null,
         "owner": "domenkozar",
         "repo": "releaser",
-        "rev": "0be40041273bd93891dd2be300d1f21f9e9a121b",
+        "rev": "52a2bb0b2ce0bc15d4e7b11d8761a28d82c0c083",
-        "sha256": "0ckgcliyi37hvpfp40nmk6r0q5irinkc2cpqs0l85z2a7si66hzh",
+        "sha256": "178lv0a0qxd8six0rm83j7wjwlsad1hysdrk4mb38fagbb8csagb",
         "type": "tarball",
-        "url": "https://github.com/domenkozar/releaser/archive/0be40041273bd93891dd2be300d1f21f9e9a121b.tar.gz",
+        "url": "https://github.com/domenkozar/releaser/archive/52a2bb0b2ce0bc15d4e7b11d8761a28d82c0c083.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

nix/sources.nix

@@ -1,29 +1,94 @@
 # This file has been generated by Niv.
 
-# A record, from name to path, of the third-party packages
+let
-with rec
-{
-  pkgs =
-    if hasNixpkgsPath
-    then
-        if hasThisAsNixpkgsPath
-        then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}
-        else import <nixpkgs> {}
-    else
-        import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {};
 
-  sources_nixpkgs =
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: spec:
+    if spec.builtin or true then
+      builtins_fetchurl { inherit (spec) url sha256; }
+    else
+      pkgs.fetchurl { inherit (spec) url sha256; };
+
+  fetch_tarball = pkgs: spec:
+    if spec.builtin or true then
+      builtins_fetchTarball { inherit (spec) url sha256; }
+    else
+      pkgs.fetchzip { inherit (spec) url sha256; };
+
+  fetch_git = spec:
+    builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; };
+
+  fetch_builtin-tarball = spec:
+    builtins.trace
+      ''
+        WARNING:
+          The niv type "builtin-tarball" will soon be deprecated. You should
+          instead use `builtin = true`.
+
+          $ niv modify <package> -a type=tarball -a builtin=true
+      ''
+      builtins_fetchTarball { inherit (spec) url sha256; };
+
+  fetch_builtin-url = spec:
+    builtins.trace
+      ''
+        WARNING:
+          The niv type "builtin-url" will soon be deprecated. You should
+          instead use `builtin = true`.
+
+          $ niv modify <package> -a type=file -a builtin=true
+      ''
+      (builtins_fetchurl { inherit (spec) url sha256; });
+
+  #
+  # Various helpers
+  #
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {};
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in
       if builtins.hasAttr "nixpkgs" sources
-    then sources.nixpkgs
+      then sourcesNixpkgs
-    else abort
+      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+        import <nixpkgs> {}
+      else
+        abort
           ''
             Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
             add a package called "nixpkgs" to your sources.json.
           '';
 
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file pkgs spec
+    else if spec.type == "tarball" then fetch_tarball pkgs spec
+    else if spec.type == "git" then fetch_git spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec
+    else if spec.type == "builtin-url" then fetch_builtin-url spec
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
   # fetchTarball version that is compatible between all the versions of Nix
-  builtins_fetchTarball =
+  builtins_fetchTarball = { url, sha256 }@attrs:
-      { url, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchTarball;
     in
@@ -33,8 +98,7 @@ with rec
         fetchTarball attrs;
 
   # fetchurl version that is compatible between all the versions of Nix
-  builtins_fetchurl =
+  builtins_fetchurl = { url, sha256 }@attrs:
-      { url, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchurl;
     in
@@ -43,51 +107,28 @@ with rec
       else
         fetchurl attrs;
 
-  # A wrapper around pkgs.fetchzip that has inspectable arguments,
+  # Create the final "sources" from the config
-  # annoyingly this means we have to specify them
+  mkSources = config:
-  fetchzip = { url, sha256 }@attrs: pkgs.fetchzip attrs;
+    mapAttrs (
-
+      name: spec:
-  # A wrapper around pkgs.fetchurl that has inspectable arguments,
-  # annoyingly this means we have to specify them
-  fetchurl = { url, sha256 }@attrs: pkgs.fetchurl attrs;
-
-  hasNixpkgsPath = (builtins.tryEval <nixpkgs>).success;
-  hasThisAsNixpkgsPath =
-    (builtins.tryEval <nixpkgs>).success && <nixpkgs> == ./.;
-
-  sources = builtins.fromJSON (builtins.readFile ./sources.json);
-
-  mapAttrs = builtins.mapAttrs or
-    (f: set: with builtins;
-      listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)));
-
-  # borrowed from nixpkgs
-  functionArgs = f: f.__functionArgs or (builtins.functionArgs f);
-  callFunctionWith = autoArgs: f: args:
-    let auto = builtins.intersectAttrs (functionArgs f) autoArgs;
-    in f (auto // args);
-
-  getFetcher = spec:
-    let fetcherName =
-      if builtins.hasAttr "type" spec
-      then builtins.getAttr "type" spec
-      else "builtin-tarball";
-    in builtins.getAttr fetcherName {
-      "tarball" = fetchzip;
-      "builtin-tarball" = builtins_fetchTarball;
-      "file" = fetchurl;
-      "builtin-url" = builtins_fetchurl;
-    };
-};
-# NOTE: spec must _not_ have an "outPath" attribute
-mapAttrs (_: spec:
         if builtins.hasAttr "outPath" spec
         then abort
           "The values in sources.json should not have an 'outPath' attribute"
         else
-    if builtins.hasAttr "url" spec && builtins.hasAttr "sha256" spec
+          spec // { outPath = fetch config.pkgs name spec; }
-    then
+    ) config.sources;
-      spec //
+
-      { outPath = callFunctionWith spec (getFetcher spec) { }; }
+  # The "config" used by the fetchers
-    else spec
+  mkConfig =
-  ) sources
+    { sourcesFile ? ./sources.json
+    , sources ? builtins.fromJSON (builtins.readFile sourcesFile)
+    , pkgs ? mkPkgs sources
+    }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+in
+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

src/haskell/lib/Arion/Images.hs

@@ -15,8 +15,6 @@ import qualified Data.Text as T
 
 import Control.Lens
 import Data.Aeson.Lens
-import System.IO (withFile, IOMode(ReadMode))
-
 
 data Image = Image
   { image :: Text -- ^ file path

tests/arion-test/default.nix

@@ -36,7 +36,7 @@ in
       pkgs.stdenv
     ];
 
-    virtualisation.memorySize = 512;
+    virtualisation.memorySize = 1024;
   };
   testScript = ''
     machine.fail("curl localhost:8000")