Merge pull request #73 from hercules-ci/various-cleanups

Various cleanups

HACKING.md

@@ -0,0 +1,31 @@
+
+# Hacking on the modules
+
+## Easiest option
+
+The module system does not distinguish between modules and configurations.
+This mean you can prototype any feature by factoring out functionality in a real-world project.
+
+## Changing the built-in modules
+
+If your change is not just an addition or if it's better implemented by refactoring, you'll want to fork and edit arion sources directly.
+
+For a fast iteration cycle (but possibly outdated arion command logic):
+
+    ~/src/arion/run-arion-quick up -d
+
+To update the arion command logic on the next run
+
+    rm ~/src/arion/result-run-arion-quick
+
+
+# Hacking on the arion command
+
+The arion command is written in Haskell. Anyone can make small changes to the code.
+Experience with Haskell tooling is not required. You can use the nixified scripts in the root of the repo for common tasks.
+ - `build` or `live-check` for typechecking
+ - `live-unit-tests` (only the test suite is "live" though)
+ - `repl` for a Haskell REPL
+ - `run-arion` to run an incrementally built arion
+ - `run-arion-via-nix` to run a nix-built arion
+ - ~~`run-arion-quick`~~ *not for command hacking;* use stale command. See previous section.

arion-compose.cabal

@@ -19,6 +19,7 @@ data-files:          nix/*.nix
                    , nix/modules/composition/*.nix
                    , nix/modules/nixos/*.nix
                    , nix/modules/service/*.nix
+                   , nix/modules/lib/*.nix
 
 -- all data is verbatim from some sources
 data-dir:            src

doc/manual/.gitignore

@@ -1,3 +1,2 @@
 manual.html
 options-composition.xml
-options-service.xml

doc/manual/Makefile

@@ -17,7 +17,7 @@ docbookxsl = http://docbook.sourceforge.net/release/xsl/current
 
 all: manual.html
 
-manual.html: manual.xml options-composition.xml options-service.xml
+manual.html: manual.xml options-composition.xml
 	$(xsltproc) --xinclude --stringparam profile.condition manual \
 	  $(docbookxsl)/profiling/profile.xsl manual.xml | \
 	  $(xsltproc) --output manual.html $(docbookxsl)/xhtml/docbook.xsl -
@@ -27,8 +27,8 @@ manual.html: manual.xml options-composition.xml options-service.xml
 	asciidoctor --backend docbook45 --doctype article $<
 	sed -e 's/<!DOCTYPE.*//' -e 's/<?asciidoc-[a-z]*?>//' -i $@
 
-options-composition.xml options-service.xml:
+options-composition.xml:
-	echo "options-composition.xml and options-service.xml should be written by the derivation. Are you running in 'nix-shell -A manual'?"; exit 1; fi
+	echo "options-composition.xml should be written by the derivation. Are you running in 'nix-shell -A manual'?"; exit 1; fi
 
 install: all
 	mkdir -p $(docdir)

doc/manual/default.nix

@@ -58,26 +58,9 @@ let
     '';
   };
 
-  serviceOptions = options {
-    moduleType = "service";
-    description = "List of Arion service-level options in JSON format";
-    optionsExpr = let
-      src = ../../src/nix;
-    in ''
-      let pkgs = import ${pkgs.path} {};
-          fixPaths = opt: opt // {
-            declarations = map (d: "src/nix" + (lib.strings.removePrefix (toString ${src}) (toString d))) opt.declarations;
-          };
-          inherit (pkgs) lib;
-          composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; name = abort "The manual's service options section must not depend on the service name."; composition = abort "The manual's service options must not depend on the composition."; };
-      in map fixPaths (lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList composition.options))
-    '';
-  };
-
   generatedDocBook = runCommand "generated-docbook" {} ''
     mkdir $out
     ln -s ${compositionOptions.optionsDocBook} $out/options-composition.xml
-    ln -s ${serviceOptions.optionsDocBook} $out/options-service.xml
   '';
 
   manual = stdenv.mkDerivation {
@@ -105,11 +88,6 @@ let
     '';
     prePatch = ''
       cp ${generatedDocBook}/* .
-      substituteInPlace options-service.xml \
-        --replace 'xml:id="appendix-configuration-options"' 'xml:id="appendix-service-options"' \
-        --replace '<title>Configuration Options</title>' '<title>Service Options</title>' \
-        --replace 'xml:id="configuration-variable-list"' 'xml:id="service-variable-list"' \
-        ;
       substituteInPlace options-composition.xml \
         --replace 'xml:id="appendix-configuration-options"' 'xml:id="appendix-composition-options"' \
         --replace '<title>Configuration Options</title>' '<title>Composition Options</title>' \

doc/manual/manual.xml

@@ -17,6 +17,5 @@
   </info>
 
   <xi:include href="options-composition.xml" />
-  <xi:include href="options-service.xml" />
 
 </book>

nix/arion.nix

@@ -11,7 +11,7 @@ let
   eval = import (srcDir + "/nix/eval-composition.nix");
   build = args@{...}:
     let composition = eval args;
-    in composition.config.build.dockerComposeYaml;
+    in composition.config.out.dockerComposeYaml;
 
 in
   justStaticExecutables (overrideCabal arion-compose (o: {

run-arion-quick

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+projectRoot="$(dirname ${BASH_SOURCE[0]})"
+resultLink="$projectRoot/result-run-arion-quick"
+
+[[ -e "$resultLink" ]] || {
+  echo 1>&2 "You don't have a prebuilt arion yet; building it."
+  nix-build "$projectRoot" -A arion --out-link "$resultLink"
+}
+
+echo 1>&2 "Note that you will need to rm '$resultLink' to rebuild the arion executable when needed."
+
+export arion_compose_datadir="$projectRoot/src"
+
+exec "$resultLink/bin/arion" "$@"

src/haskell/exe/Main.hs

@@ -183,9 +183,7 @@ runRepl co = do
     "Launching a repl for you. To get started:\n\
     \\n\
     \To see deployment-wide configuration\n\
-    \  type config. and hit TAB\n\
+    \  type config. and use tab completion\n\
-    \To see the services\n\
-    \  type config.docker-compose.evaluatedServices TAB or ENTER\n\
     \To bring the top-level Nixpkgs attributes into scope\n\
     \  type :a (config._module.args.pkgs) // { inherit config; }\n\
     \"

src/haskell/lib/Arion/Nix.hs

@@ -51,7 +51,7 @@ evaluateComposition ea = do
         , "--strict"
         , "--json"
         , "--attr"
-        , "config.build.dockerComposeYamlAttrs"
+        , "config.out.dockerComposeYamlAttrs"
         ]
       args =
         [ evalComposition ]
@@ -99,7 +99,7 @@ buildComposition outLink ea = do
   evalComposition <- getEvalCompositionFile
   let commandArgs =
         [ "--attr"
-        , "config.build.dockerComposeYaml"
+        , "config.out.dockerComposeYaml"
         , "--out-link"
         , outLink
         ]

src/nix/eval-composition.nix

@@ -34,5 +34,9 @@ let
   };
 
 in
-  # Typically you need composition.config.build.dockerComposeYaml
+  # Typically you need composition.config.out.dockerComposeYaml
-  composition
+  composition // {
+    # throw in lib and pkgs for repl convenience
+    inherit lib;
+    inherit (composition.config._module.args) pkgs;
+  }

src/nix/eval-service.nix

@@ -1,32 +0,0 @@
-{ lib, pkgs, ... }:
-
-{ modules, host, name, composition }:
-let
-  composite = lib.evalModules {
-    check = true;
-    modules = builtinModules ++ modules;
-  };
-
-  builtinModules = [
-    argsModule
-    ./modules/service/default-exec.nix
-    ./modules/service/docker-compose-service.nix
-    ./modules/service/extended-info.nix
-    ./modules/service/host-store.nix
-    ./modules/service/context.nix
-    ./modules/service/image.nix
-    ./modules/service/nixos.nix
-    ./modules/service/nixos-init.nix
-  ];
-
-  argsModule = {
-    _file = ./eval-service.nix;
-    key = ./eval-service.nix;
-    config._module.args.pkgs = lib.mkForce pkgs;
-    config.host = host;
-    config.service.name = name;
-    config.composition = composition;
-  };
-
-in
-  composite

src/nix/modules/composition/arion-base-image.nix

@@ -34,7 +34,7 @@ in
 
   config = {
     arionBaseImage = "${name}:${tag}";
-    build.imagesToLoad = lib.mkIf (lib.any (s: s.config.service.useHostStore) (lib.attrValues config.docker-compose.evaluatedServices)) [
+    build.imagesToLoad = lib.mkIf (lib.any (s: s.service.useHostStore) (lib.attrValues config.services)) [
       { image = builtImage; imageName = name; imageTag = tag; }
     ];  
   };

src/nix/modules/composition/docker-compose.nix

@@ -3,36 +3,52 @@
    This is a composition-level module.
 
    It defines the low-level options that are read by arion, like
-    - build.dockerComposeYaml
+    - out.dockerComposeYaml
 
    It declares options like
-    - docker-compose.services
+    - services
 
  */
-{ pkgs, lib, config, ... }:
+compositionArgs@{ lib, config, options, pkgs, ... }:
 let
-  evalService = name: modules: pkgs.callPackage ../../eval-service.nix {} {
+  inherit (lib) types;
-    inherit name modules;
+
-    inherit (config) host;
+  service = {
-    composition = config;
+    imports = [ argsModule ] ++ import ../service/all-modules.nix;
   };
+  argsModule =
+    { name, # injected by types.submodule
+      ...
+    }: {
+      _file = ./docker-compose.nix;
+      key = ./docker-compose.nix;
+
+      config._module.args.pkgs = lib.mkDefault compositionArgs.pkgs;
+      config.host = compositionArgs.config.host;
+      config.composition = compositionArgs.config;
+      config.service.name = name;
+    };
 
 in
 {
+  imports = [
+    ../lib/assert.nix
+    (lib.mkRenamedOptionModule ["docker-compose" "services"] ["services"])
+  ];
   options = {
-    build.dockerComposeYaml = lib.mkOption {
+    out.dockerComposeYaml = lib.mkOption {
       type = lib.types.package;
       description = "A derivation that produces a docker-compose.yaml file for this composition.";
       readOnly = true;
     };
-    build.dockerComposeYamlText = lib.mkOption {
+    out.dockerComposeYamlText = lib.mkOption {
       type = lib.types.str;
-      description = "The text of build.dockerComposeYaml.";
+      description = "The text of out.dockerComposeYaml.";
       readOnly = true;
     };
-    build.dockerComposeYamlAttrs = lib.mkOption {
+    out.dockerComposeYamlAttrs = lib.mkOption {
       type = lib.types.attrsOf lib.types.unspecified;
-      description = "The text of build.dockerComposeYaml.";
+      description = "The text of out.dockerComposeYaml.";
       readOnly = true;
     };
     docker-compose.raw = lib.mkOption {
@@ -43,26 +59,19 @@ in
       type = lib.types.attrs;
       description = "Attribute set that will be turned into the x-arion section of the docker-compose.yaml file.";
     };
-    docker-compose.services = lib.mkOption {
+    services = lib.mkOption {
-      default = {};
+      type = lib.types.attrsOf (lib.types.submodule service);
-      type = with lib.types; attrsOf (coercedTo unspecified (a: [a]) (listOf unspecified));
+      description = "An attribute set of service configurations. A service specifies how to run an image as a container.";
-      description = "A attribute set of service configurations. A service specifies how to run an image. Each of these service configurations is specified using modules whose options are described in the Service Options section.";
-    };
-    docker-compose.evaluatedServices = lib.mkOption {
-      type = lib.types.attrsOf lib.types.attrs;
-      description = "Attribute set of evaluated service configurations.";
-      readOnly = true;
     };
   };
   config = {
-    build.dockerComposeYaml = pkgs.writeText "docker-compose.yaml" config.build.dockerComposeYamlText;
+    out.dockerComposeYaml = pkgs.writeText "docker-compose.yaml" config.out.dockerComposeYamlText;
-    build.dockerComposeYamlText = builtins.toJSON (config.build.dockerComposeYamlAttrs);
+    out.dockerComposeYamlText = builtins.toJSON (config.out.dockerComposeYamlAttrs);
-    build.dockerComposeYamlAttrs = config.docker-compose.raw;
+    out.dockerComposeYamlAttrs = config.assertWarn config.docker-compose.raw;
 
-    docker-compose.evaluatedServices = lib.mapAttrs evalService config.docker-compose.services;
     docker-compose.raw = {
       version = "3.4";
-      services = lib.mapAttrs (k: c: c.config.build.service) config.docker-compose.evaluatedServices;
+      services = lib.mapAttrs (k: c: c.out.service) config.services;
       x-arion = config.docker-compose.extended;
     };
   };

src/nix/modules/composition/images.nix

@@ -4,17 +4,17 @@ let
 
   serviceImages =
     lib.mapAttrs addDetails (
-      lib.filterAttrs filterFunction config.docker-compose.evaluatedServices
+      lib.filterAttrs filterFunction config.services
     );
 
   filterFunction = serviceName: service:
     builtins.addErrorContext "while evaluating whether the service ${serviceName} defines an image"
-      service.config.image.nixBuild;
+      service.image.nixBuild;
 
   addDetails = serviceName: service:
     builtins.addErrorContext "while evaluating the image for service ${serviceName}"
     (let
-      inherit (service.config) build;
+      inherit (service) build;
     in {
       image = build.image.outPath;
       imageName = build.imageName or service.image.name;
@@ -28,6 +28,7 @@ in
   options = {
     build.imagesToLoad = lib.mkOption {
       type = listOf unspecified;
+      internal = true;
       description = "List of dockerTools image derivations.";
     };
   };

src/nix/modules/composition/service-info.nix

@@ -5,16 +5,13 @@
  */
 { config, lib, ... }:
 let
+  inherit (lib) mapAttrs filterAttrs;
+
   serviceInfo =
-    lib.mapAttrs getInfo (
+    filterAttrs (_k: v: v != {})
-      lib.filterAttrs filterFunction config.docker-compose.evaluatedServices
+      (mapAttrs (_serviceName: service: service.out.extendedInfo)
-    );
+        config.services
-
+      );
-  filterFunction = _serviceName: service:
-    # shallow equality suffices for emptiness test
-    builtins.attrNames service.config.build.extendedInfo != [];
-
-  getInfo = _serviceName: service: service.config.build.extendedInfo;
 
 in
 {

src/nix/modules/lib/README.md

@@ -0,0 +1,2 @@
+`assertions.nix`: Nixpkgs
+`assert.nix`: extracted from Nixpkgs, adapted

src/nix/modules/lib/assert.nix

@@ -0,0 +1,32 @@
+{ config, lib, pkgs, ... }:
+
+# based on nixpkgs/nixos/modules/system/activation/top-level.nix
+
+let
+  inherit (lib) filter concatStringsSep types mkOption;
+
+  # lib.showWarnings since 19.09
+  showWarnings = warnings: res: lib.fold (w: x: lib.warn w x) res warnings;
+  warn = msg: builtins.trace "warning: ${msg}";
+
+  # Handle assertions and warnings
+
+  failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions);
+
+  assertWarn = if failedAssertions != []
+    then throw "\nFailed assertions:\n${concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}"
+    else showWarnings config.warnings;
+
+in
+
+{
+  imports = [ ./assertions.nix ];
+  options.assertWarn = mkOption {
+    type = types.unspecified; # a function
+    # It's for the wrapping program to know about this. User need not care.
+    internal = true;
+    readOnly = true;
+  };
+  config = { inherit assertWarn; };
+}
+

src/nix/modules/lib/assertions.nix

@@ -0,0 +1,34 @@
+{ lib, ... }:
+
+with lib;
+
+{
+
+  options = {
+
+    assertions = mkOption {
+      type = types.listOf types.unspecified;
+      internal = true;
+      default = [];
+      example = [ { assertion = false; message = "you can't enable this for that reason"; } ];
+      description = ''
+        This option allows modules to express conditions that must
+        hold for the evaluation of the system configuration to
+        succeed, along with associated error messages for the user.
+      '';
+    };
+
+    warnings = mkOption {
+      internal = true;
+      default = [];
+      type = types.listOf types.str;
+      example = [ "The `foo' service is deprecated and will go away soon!" ];
+      description = ''
+        This option allows modules to show warnings to users during
+        the evaluation of the system configuration.
+      '';
+    };
+
+  };
+  # impl of assertions is in <nixpkgs/nixos/modules/system/activation/top-level.nix>
+}

src/nix/modules/service/all-modules.nix

@@ -0,0 +1,11 @@
+[
+  ./default-exec.nix
+  ./docker-compose-service.nix
+  ./extended-info.nix
+  ./host-store.nix
+  ./context.nix
+  ./image.nix
+  ./nixos.nix
+  ./nixos-init.nix
+  ../lib/assert.nix
+]

src/nix/modules/service/context.nix

@@ -3,12 +3,14 @@
   options = {
     host = lib.mkOption {
       type = lib.types.attrs;
+      readOnly = true;
       description = ''
         The composition-level host option values.
       '';
     };
     composition = lib.mkOption {
       type = lib.types.attrs;
+      readOnly = true;
       description = ''
         The composition configuration.
       '';

src/nix/modules/service/default-exec.nix

@@ -14,6 +14,6 @@ in
     };
   };
   config = {
-    build.extendedInfo.defaultExec = config.service.defaultExec;
+    out.extendedInfo.defaultExec = config.service.defaultExec;
   };
 }

src/nix/modules/service/docker-compose-service.nix

@@ -1,6 +1,6 @@
 /*
 
-   This service-level module defines the build.service option, using
+   This service-level module defines the out.service option, using
    the user-facing options service.image, service.volumes, etc.
 
  */
@@ -25,8 +25,12 @@ let
 
 in
 {
+  imports = [
+    (lib.mkRenamedOptionModule ["build" "service"] ["out" "service"])
+  ];
+
   options = {
-    build.service = mkOption {
+    out.service = mkOption {
       type = attrsOf types.unspecified;
       description = ''
         Raw input for the service in <code>docker-compose.yaml</code>.
@@ -37,12 +41,13 @@ in
         This option is user accessible because it may serve as an
         escape hatch for some.
       '';
+      apply = config.assertWarn;
     };
 
     service.name = mkOption {
       type = str;
       description = ''
-        The name of the service - <code>&lt;name></code> in the composition-level <code>docker-compose.services.&lt;name></code>
+        The name of the service - <code>&lt;name></code> in the composition-level <code>services.&lt;name></code>
       '';
       readOnly = true;
     };
@@ -209,7 +214,7 @@ in
     };
   };
 
-  config.build.service = {
+  config.out.service = {
     inherit (config.service)
       volumes
       environment

src/nix/modules/service/extended-info.nix

@@ -4,8 +4,11 @@ let
   inherit (lib.types) attrsOf unspecified;
 in
 {
+  imports = [
+    (lib.mkRenamedOptionModule ["build" "extendedInfo"] ["out" "extendedInfo"])
+  ];
   options = {
-    build.extendedInfo = mkOption {
+    out.extendedInfo = mkOption {
       type = attrsOf unspecified;
       description = ''
         Information about a service to include in the Docker Compose file,

tests/arion-test/default.nix

@@ -30,9 +30,9 @@ in
     virtualisation.pathsInNixDB = [
       # Pre-build the image because we don't want to build the world
       # in the vm.
-      (preEval [ ../../examples/minimal/arion-compose.nix ]).config.build.dockerComposeYaml
+      (preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml
-      (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.build.dockerComposeYaml
+      (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml
-      (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.build.dockerComposeYaml
+      (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml
       pkgs.stdenv
     ];
   };