diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..542db67 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,31 @@ + +# Hacking on the modules + +## Easiest option + +The module system does not distinguish between modules and configurations. +This mean you can prototype any feature by factoring out functionality in a real-world project. + +## Changing the built-in modules + +If your change is not just an addition or if it's better implemented by refactoring, you'll want to fork and edit arion sources directly. + +For a fast iteration cycle (but possibly outdated arion command logic): + + ~/src/arion/run-arion-quick up -d + +To update the arion command logic on the next run + + rm ~/src/arion/result-run-arion-quick + + +# Hacking on the arion command + +The arion command is written in Haskell. Anyone can make small changes to the code. +Experience with Haskell tooling is not required. You can use the nixified scripts in the root of the repo for common tasks. + - `build` or `live-check` for typechecking + - `live-unit-tests` (only the test suite is "live" though) + - `repl` for a Haskell REPL + - `run-arion` to run an incrementally built arion + - `run-arion-via-nix` to run a nix-built arion + - ~~`run-arion-quick`~~ *not for command hacking;* use stale command. See previous section. diff --git a/arion-compose.cabal b/arion-compose.cabal index 83aa6f6..2ff2f0b 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -19,6 +19,7 @@ data-files: nix/*.nix , nix/modules/composition/*.nix , nix/modules/nixos/*.nix , nix/modules/service/*.nix + , nix/modules/lib/*.nix -- all data is verbatim from some sources data-dir: src diff --git a/doc/manual/.gitignore b/doc/manual/.gitignore index cfc155d..0c15756 100644 --- a/doc/manual/.gitignore +++ b/doc/manual/.gitignore @@ -1,3 +1,2 @@ manual.html options-composition.xml -options-service.xml diff --git a/doc/manual/Makefile b/doc/manual/Makefile index cc31da1..656c0ab 100644 --- a/doc/manual/Makefile +++ b/doc/manual/Makefile @@ -17,7 +17,7 @@ docbookxsl = http://docbook.sourceforge.net/release/xsl/current all: manual.html -manual.html: manual.xml options-composition.xml options-service.xml +manual.html: manual.xml options-composition.xml $(xsltproc) --xinclude --stringparam profile.condition manual \ $(docbookxsl)/profiling/profile.xsl manual.xml | \ $(xsltproc) --output manual.html $(docbookxsl)/xhtml/docbook.xsl - @@ -27,8 +27,8 @@ manual.html: manual.xml options-composition.xml options-service.xml asciidoctor --backend docbook45 --doctype article $< sed -e 's///' -i $@ -options-composition.xml options-service.xml: - echo "options-composition.xml and options-service.xml should be written by the derivation. Are you running in 'nix-shell -A manual'?"; exit 1; fi +options-composition.xml: + echo "options-composition.xml should be written by the derivation. Are you running in 'nix-shell -A manual'?"; exit 1; fi install: all mkdir -p $(docdir) diff --git a/doc/manual/default.nix b/doc/manual/default.nix index 03522ab..64b352b 100644 --- a/doc/manual/default.nix +++ b/doc/manual/default.nix @@ -58,26 +58,9 @@ let ''; }; - serviceOptions = options { - moduleType = "service"; - description = "List of Arion service-level options in JSON format"; - optionsExpr = let - src = ../../src/nix; - in '' - let pkgs = import ${pkgs.path} {}; - fixPaths = opt: opt // { - declarations = map (d: "src/nix" + (lib.strings.removePrefix (toString ${src}) (toString d))) opt.declarations; - }; - inherit (pkgs) lib; - composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; name = abort "The manual's service options section must not depend on the service name."; composition = abort "The manual's service options must not depend on the composition."; }; - in map fixPaths (lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList composition.options)) - ''; - }; - generatedDocBook = runCommand "generated-docbook" {} '' mkdir $out ln -s ${compositionOptions.optionsDocBook} $out/options-composition.xml - ln -s ${serviceOptions.optionsDocBook} $out/options-service.xml ''; manual = stdenv.mkDerivation { @@ -105,11 +88,6 @@ let ''; prePatch = '' cp ${generatedDocBook}/* . - substituteInPlace options-service.xml \ - --replace 'xml:id="appendix-configuration-options"' 'xml:id="appendix-service-options"' \ - --replace 'Configuration Options' 'Service Options' \ - --replace 'xml:id="configuration-variable-list"' 'xml:id="service-variable-list"' \ - ; substituteInPlace options-composition.xml \ --replace 'xml:id="appendix-configuration-options"' 'xml:id="appendix-composition-options"' \ --replace 'Configuration Options' 'Composition Options' \ diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml index bce99c2..87d9616 100644 --- a/doc/manual/manual.xml +++ b/doc/manual/manual.xml @@ -17,6 +17,5 @@ - diff --git a/nix/arion.nix b/nix/arion.nix index 4b76fc1..b53b1f4 100644 --- a/nix/arion.nix +++ b/nix/arion.nix @@ -11,7 +11,7 @@ let eval = import (srcDir + "/nix/eval-composition.nix"); build = args@{...}: let composition = eval args; - in composition.config.build.dockerComposeYaml; + in composition.config.out.dockerComposeYaml; in justStaticExecutables (overrideCabal arion-compose (o: { diff --git a/run-arion-quick b/run-arion-quick new file mode 100755 index 0000000..df1c3e8 --- /dev/null +++ b/run-arion-quick @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +projectRoot="$(dirname ${BASH_SOURCE[0]})" +resultLink="$projectRoot/result-run-arion-quick" + +[[ -e "$resultLink" ]] || { + echo 1>&2 "You don't have a prebuilt arion yet; building it." + nix-build "$projectRoot" -A arion --out-link "$resultLink" +} + +echo 1>&2 "Note that you will need to rm '$resultLink' to rebuild the arion executable when needed." + +export arion_compose_datadir="$projectRoot/src" + +exec "$resultLink/bin/arion" "$@" diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index 21ec3d3..57fcada 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -183,9 +183,7 @@ runRepl co = do "Launching a repl for you. To get started:\n\ \\n\ \To see deployment-wide configuration\n\ - \ type config. and hit TAB\n\ - \To see the services\n\ - \ type config.docker-compose.evaluatedServices TAB or ENTER\n\ + \ type config. and use tab completion\n\ \To bring the top-level Nixpkgs attributes into scope\n\ \ type :a (config._module.args.pkgs) // { inherit config; }\n\ \" diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index cef6a92..a90343a 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -51,7 +51,7 @@ evaluateComposition ea = do , "--strict" , "--json" , "--attr" - , "config.build.dockerComposeYamlAttrs" + , "config.out.dockerComposeYamlAttrs" ] args = [ evalComposition ] @@ -99,7 +99,7 @@ buildComposition outLink ea = do evalComposition <- getEvalCompositionFile let commandArgs = [ "--attr" - , "config.build.dockerComposeYaml" + , "config.out.dockerComposeYaml" , "--out-link" , outLink ] diff --git a/src/nix/eval-composition.nix b/src/nix/eval-composition.nix index d4a2b57..1081864 100644 --- a/src/nix/eval-composition.nix +++ b/src/nix/eval-composition.nix @@ -34,5 +34,9 @@ let }; in - # Typically you need composition.config.build.dockerComposeYaml - composition + # Typically you need composition.config.out.dockerComposeYaml + composition // { + # throw in lib and pkgs for repl convenience + inherit lib; + inherit (composition.config._module.args) pkgs; + } diff --git a/src/nix/eval-service.nix b/src/nix/eval-service.nix deleted file mode 100644 index 2ab731b..0000000 --- a/src/nix/eval-service.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ lib, pkgs, ... }: - -{ modules, host, name, composition }: -let - composite = lib.evalModules { - check = true; - modules = builtinModules ++ modules; - }; - - builtinModules = [ - argsModule - ./modules/service/default-exec.nix - ./modules/service/docker-compose-service.nix - ./modules/service/extended-info.nix - ./modules/service/host-store.nix - ./modules/service/context.nix - ./modules/service/image.nix - ./modules/service/nixos.nix - ./modules/service/nixos-init.nix - ]; - - argsModule = { - _file = ./eval-service.nix; - key = ./eval-service.nix; - config._module.args.pkgs = lib.mkForce pkgs; - config.host = host; - config.service.name = name; - config.composition = composition; - }; - -in - composite diff --git a/src/nix/modules/composition/arion-base-image.nix b/src/nix/modules/composition/arion-base-image.nix index fad7b6c..132bc4c 100644 --- a/src/nix/modules/composition/arion-base-image.nix +++ b/src/nix/modules/composition/arion-base-image.nix @@ -34,7 +34,7 @@ in config = { arionBaseImage = "${name}:${tag}"; - build.imagesToLoad = lib.mkIf (lib.any (s: s.config.service.useHostStore) (lib.attrValues config.docker-compose.evaluatedServices)) [ + build.imagesToLoad = lib.mkIf (lib.any (s: s.service.useHostStore) (lib.attrValues config.services)) [ { image = builtImage; imageName = name; imageTag = tag; } ]; }; diff --git a/src/nix/modules/composition/docker-compose.nix b/src/nix/modules/composition/docker-compose.nix index 1891bbf..a74c958 100644 --- a/src/nix/modules/composition/docker-compose.nix +++ b/src/nix/modules/composition/docker-compose.nix @@ -3,36 +3,52 @@ This is a composition-level module. It defines the low-level options that are read by arion, like - - build.dockerComposeYaml + - out.dockerComposeYaml It declares options like - - docker-compose.services + - services */ -{ pkgs, lib, config, ... }: +compositionArgs@{ lib, config, options, pkgs, ... }: let - evalService = name: modules: pkgs.callPackage ../../eval-service.nix {} { - inherit name modules; - inherit (config) host; - composition = config; + inherit (lib) types; + + service = { + imports = [ argsModule ] ++ import ../service/all-modules.nix; }; + argsModule = + { name, # injected by types.submodule + ... + }: { + _file = ./docker-compose.nix; + key = ./docker-compose.nix; + + config._module.args.pkgs = lib.mkDefault compositionArgs.pkgs; + config.host = compositionArgs.config.host; + config.composition = compositionArgs.config; + config.service.name = name; + }; in { + imports = [ + ../lib/assert.nix + (lib.mkRenamedOptionModule ["docker-compose" "services"] ["services"]) + ]; options = { - build.dockerComposeYaml = lib.mkOption { + out.dockerComposeYaml = lib.mkOption { type = lib.types.package; description = "A derivation that produces a docker-compose.yaml file for this composition."; readOnly = true; }; - build.dockerComposeYamlText = lib.mkOption { + out.dockerComposeYamlText = lib.mkOption { type = lib.types.str; - description = "The text of build.dockerComposeYaml."; + description = "The text of out.dockerComposeYaml."; readOnly = true; }; - build.dockerComposeYamlAttrs = lib.mkOption { + out.dockerComposeYamlAttrs = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; - description = "The text of build.dockerComposeYaml."; + description = "The text of out.dockerComposeYaml."; readOnly = true; }; docker-compose.raw = lib.mkOption { @@ -43,26 +59,19 @@ in type = lib.types.attrs; description = "Attribute set that will be turned into the x-arion section of the docker-compose.yaml file."; }; - docker-compose.services = lib.mkOption { - default = {}; - type = with lib.types; attrsOf (coercedTo unspecified (a: [a]) (listOf unspecified)); - description = "A attribute set of service configurations. A service specifies how to run an image. Each of these service configurations is specified using modules whose options are described in the Service Options section."; - }; - docker-compose.evaluatedServices = lib.mkOption { - type = lib.types.attrsOf lib.types.attrs; - description = "Attribute set of evaluated service configurations."; - readOnly = true; + services = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule service); + description = "An attribute set of service configurations. A service specifies how to run an image as a container."; }; }; config = { - build.dockerComposeYaml = pkgs.writeText "docker-compose.yaml" config.build.dockerComposeYamlText; - build.dockerComposeYamlText = builtins.toJSON (config.build.dockerComposeYamlAttrs); - build.dockerComposeYamlAttrs = config.docker-compose.raw; + out.dockerComposeYaml = pkgs.writeText "docker-compose.yaml" config.out.dockerComposeYamlText; + out.dockerComposeYamlText = builtins.toJSON (config.out.dockerComposeYamlAttrs); + out.dockerComposeYamlAttrs = config.assertWarn config.docker-compose.raw; - docker-compose.evaluatedServices = lib.mapAttrs evalService config.docker-compose.services; docker-compose.raw = { version = "3.4"; - services = lib.mapAttrs (k: c: c.config.build.service) config.docker-compose.evaluatedServices; + services = lib.mapAttrs (k: c: c.out.service) config.services; x-arion = config.docker-compose.extended; }; }; diff --git a/src/nix/modules/composition/images.nix b/src/nix/modules/composition/images.nix index bedf8e8..dc93691 100644 --- a/src/nix/modules/composition/images.nix +++ b/src/nix/modules/composition/images.nix @@ -4,17 +4,17 @@ let serviceImages = lib.mapAttrs addDetails ( - lib.filterAttrs filterFunction config.docker-compose.evaluatedServices + lib.filterAttrs filterFunction config.services ); filterFunction = serviceName: service: builtins.addErrorContext "while evaluating whether the service ${serviceName} defines an image" - service.config.image.nixBuild; + service.image.nixBuild; addDetails = serviceName: service: builtins.addErrorContext "while evaluating the image for service ${serviceName}" (let - inherit (service.config) build; + inherit (service) build; in { image = build.image.outPath; imageName = build.imageName or service.image.name; @@ -28,6 +28,7 @@ in options = { build.imagesToLoad = lib.mkOption { type = listOf unspecified; + internal = true; description = "List of dockerTools image derivations."; }; }; diff --git a/src/nix/modules/composition/service-info.nix b/src/nix/modules/composition/service-info.nix index 1ed2989..6c66a30 100644 --- a/src/nix/modules/composition/service-info.nix +++ b/src/nix/modules/composition/service-info.nix @@ -5,16 +5,13 @@ */ { config, lib, ... }: let + inherit (lib) mapAttrs filterAttrs; + serviceInfo = - lib.mapAttrs getInfo ( - lib.filterAttrs filterFunction config.docker-compose.evaluatedServices - ); - - filterFunction = _serviceName: service: - # shallow equality suffices for emptiness test - builtins.attrNames service.config.build.extendedInfo != []; - - getInfo = _serviceName: service: service.config.build.extendedInfo; + filterAttrs (_k: v: v != {}) + (mapAttrs (_serviceName: service: service.out.extendedInfo) + config.services + ); in { diff --git a/src/nix/modules/lib/README.md b/src/nix/modules/lib/README.md new file mode 100644 index 0000000..48a0c04 --- /dev/null +++ b/src/nix/modules/lib/README.md @@ -0,0 +1,2 @@ +`assertions.nix`: Nixpkgs +`assert.nix`: extracted from Nixpkgs, adapted diff --git a/src/nix/modules/lib/assert.nix b/src/nix/modules/lib/assert.nix new file mode 100644 index 0000000..f9bf4a8 --- /dev/null +++ b/src/nix/modules/lib/assert.nix @@ -0,0 +1,32 @@ +{ config, lib, pkgs, ... }: + +# based on nixpkgs/nixos/modules/system/activation/top-level.nix + +let + inherit (lib) filter concatStringsSep types mkOption; + + # lib.showWarnings since 19.09 + showWarnings = warnings: res: lib.fold (w: x: lib.warn w x) res warnings; + warn = msg: builtins.trace "warning: ${msg}"; + + # Handle assertions and warnings + + failedAssertions = map (x: x.message) (filter (x: !x.assertion) config.assertions); + + assertWarn = if failedAssertions != [] + then throw "\nFailed assertions:\n${concatStringsSep "\n" (map (x: "- ${x}") failedAssertions)}" + else showWarnings config.warnings; + +in + +{ + imports = [ ./assertions.nix ]; + options.assertWarn = mkOption { + type = types.unspecified; # a function + # It's for the wrapping program to know about this. User need not care. + internal = true; + readOnly = true; + }; + config = { inherit assertWarn; }; +} + diff --git a/src/nix/modules/lib/assertions.nix b/src/nix/modules/lib/assertions.nix new file mode 100644 index 0000000..550b3ac --- /dev/null +++ b/src/nix/modules/lib/assertions.nix @@ -0,0 +1,34 @@ +{ lib, ... }: + +with lib; + +{ + + options = { + + assertions = mkOption { + type = types.listOf types.unspecified; + internal = true; + default = []; + example = [ { assertion = false; message = "you can't enable this for that reason"; } ]; + description = '' + This option allows modules to express conditions that must + hold for the evaluation of the system configuration to + succeed, along with associated error messages for the user. + ''; + }; + + warnings = mkOption { + internal = true; + default = []; + type = types.listOf types.str; + example = [ "The `foo' service is deprecated and will go away soon!" ]; + description = '' + This option allows modules to show warnings to users during + the evaluation of the system configuration. + ''; + }; + + }; + # impl of assertions is in +} diff --git a/src/nix/modules/service/all-modules.nix b/src/nix/modules/service/all-modules.nix new file mode 100644 index 0000000..18c0432 --- /dev/null +++ b/src/nix/modules/service/all-modules.nix @@ -0,0 +1,11 @@ +[ + ./default-exec.nix + ./docker-compose-service.nix + ./extended-info.nix + ./host-store.nix + ./context.nix + ./image.nix + ./nixos.nix + ./nixos-init.nix + ../lib/assert.nix +] diff --git a/src/nix/modules/service/context.nix b/src/nix/modules/service/context.nix index 10854e2..f80eaa3 100644 --- a/src/nix/modules/service/context.nix +++ b/src/nix/modules/service/context.nix @@ -3,12 +3,14 @@ options = { host = lib.mkOption { type = lib.types.attrs; + readOnly = true; description = '' The composition-level host option values. ''; }; composition = lib.mkOption { type = lib.types.attrs; + readOnly = true; description = '' The composition configuration. ''; diff --git a/src/nix/modules/service/default-exec.nix b/src/nix/modules/service/default-exec.nix index 299e083..d3c31b3 100644 --- a/src/nix/modules/service/default-exec.nix +++ b/src/nix/modules/service/default-exec.nix @@ -14,6 +14,6 @@ in }; }; config = { - build.extendedInfo.defaultExec = config.service.defaultExec; + out.extendedInfo.defaultExec = config.service.defaultExec; }; } \ No newline at end of file diff --git a/src/nix/modules/service/docker-compose-service.nix b/src/nix/modules/service/docker-compose-service.nix index 80a795d..0451b78 100644 --- a/src/nix/modules/service/docker-compose-service.nix +++ b/src/nix/modules/service/docker-compose-service.nix @@ -1,6 +1,6 @@ /* - This service-level module defines the build.service option, using + This service-level module defines the out.service option, using the user-facing options service.image, service.volumes, etc. */ @@ -25,8 +25,12 @@ let in { + imports = [ + (lib.mkRenamedOptionModule ["build" "service"] ["out" "service"]) + ]; + options = { - build.service = mkOption { + out.service = mkOption { type = attrsOf types.unspecified; description = '' Raw input for the service in docker-compose.yaml. @@ -37,12 +41,13 @@ in This option is user accessible because it may serve as an escape hatch for some. ''; + apply = config.assertWarn; }; service.name = mkOption { type = str; description = '' - The name of the service - <name> in the composition-level docker-compose.services.<name> + The name of the service - <name> in the composition-level services.<name> ''; readOnly = true; }; @@ -209,7 +214,7 @@ in }; }; - config.build.service = { + config.out.service = { inherit (config.service) volumes environment diff --git a/src/nix/modules/service/extended-info.nix b/src/nix/modules/service/extended-info.nix index bb876c7..c6a35f0 100644 --- a/src/nix/modules/service/extended-info.nix +++ b/src/nix/modules/service/extended-info.nix @@ -4,8 +4,11 @@ let inherit (lib.types) attrsOf unspecified; in { + imports = [ + (lib.mkRenamedOptionModule ["build" "extendedInfo"] ["out" "extendedInfo"]) + ]; options = { - build.extendedInfo = mkOption { + out.extendedInfo = mkOption { type = attrsOf unspecified; description = '' Information about a service to include in the Docker Compose file, diff --git a/tests/arion-test/default.nix b/tests/arion-test/default.nix index e197fb2..caf892a 100644 --- a/tests/arion-test/default.nix +++ b/tests/arion-test/default.nix @@ -30,9 +30,9 @@ in virtualisation.pathsInNixDB = [ # Pre-build the image because we don't want to build the world # in the vm. - (preEval [ ../../examples/minimal/arion-compose.nix ]).config.build.dockerComposeYaml - (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.build.dockerComposeYaml - (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.build.dockerComposeYaml + (preEval [ ../../examples/minimal/arion-compose.nix ]).config.out.dockerComposeYaml + (preEval [ ../../examples/full-nixos/arion-compose.nix ]).config.out.dockerComposeYaml + (preEval [ ../../examples/nixos-unit/arion-compose.nix ]).config.out.dockerComposeYaml pkgs.stdenv ]; };