From ccaac02a87b23dcced02f2e3187047ce88de1351 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 3 Mar 2019 23:42:40 +0100 Subject: [PATCH] Add doc/manual, tweak README --- README.md | 24 +- default.nix | 1 + doc/default.nix | 8 + doc/manual/.gitignore | 4 + doc/manual/Makefile | 35 +++ doc/manual/default.nix | 120 ++++++++++ doc/manual/installation.asciidoc | 37 +++ doc/manual/introduction.asciidoc | 33 +++ doc/manual/live-build | 3 + doc/manual/manual.xml | 25 ++ doc/manual/options-to-docbook.xsl | 224 ++++++++++++++++++ doc/manual/style.css | 159 +++++++++++++ nix/overlay.nix | 1 + src/nix/eval-service.nix | 25 ++ .../modules/composition/docker-compose.nix | 29 +-- .../modules/composition/host-environment.nix | 5 +- .../service/docker-compose-service.nix | 39 ++- 17 files changed, 731 insertions(+), 41 deletions(-) create mode 100644 doc/default.nix create mode 100644 doc/manual/.gitignore create mode 100644 doc/manual/Makefile create mode 100644 doc/manual/default.nix create mode 100644 doc/manual/installation.asciidoc create mode 100644 doc/manual/introduction.asciidoc create mode 100755 doc/manual/live-build create mode 100644 doc/manual/manual.xml create mode 100644 doc/manual/options-to-docbook.xsl create mode 100644 doc/manual/style.css create mode 100644 src/nix/eval-service.nix diff --git a/README.md b/README.md index 2bdef10..996f469 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,9 @@ *Wait, what?* With Arion you can fire up containers without creating images for each -service. Instead, it uses a mostly empty image, sort of like a base -image, and makes the host's Nix store available in the container, -allowing the container to run programs without having to re-package -them into a docker image. +service. It can use a mostly empty image, and makes the host's Nix +store available in the container, allowing the container to run +programs without having to re-package them into a docker image. Arion is configured using Nix with modules, like those in NixOS. Similar to `docker-compose` it can therefore combine @@ -21,13 +20,10 @@ development environments while working on [Hercules CI](https://www.hercules-ci.com). (It was also born out of ancient Greek deities disguised as horses. More on that later.) -We don't use it to deploy to 'real' environments and we have no plans -to do so in the future. - -If you do want to use Arion for 'real' environments, you'll probably -want to either build images or manage garbage collection roots if you -control the deployment host. Either of these has yet to be -implemented. +If you do want to use Arion for production environments, you'll +probably want to either build normal container images or manage +garbage collection roots if you control the deployment host. Neither +scenario is made easier by arion at this time. Support for other Linux than NixOS is untested. @@ -68,7 +64,7 @@ This Nix expression serves the Nix manual at host port 8000 when launched with ` } ``` -The `pkgs` argument comes from a file called `arion-pkgs.nix`. It can be as simple as `import {}` to use the Nixpkgs from your `$NIX_PATH`. +The `pkgs` argument comes from a file called `arion-pkgs.nix`. It can be as simple as `import {}` to use the Nixpkgs from your `$NIX_PATH`, or you can use it to pin a specific Nixpkgs version. # A full featured example @@ -90,7 +86,7 @@ When it runs, it does the following: Most of the interesting stuff happens in Arion's Nix expressions, where it runs the module system (known from NixOS) and provides the configuration that makes the Docker Compose file do the things it needs to do. -The interesting part is of course the [service-host-store.nix module](src/nix/service-host-store.nix) which performs the bind mounts to make the host Nix store available in the container. +One of the more interesting built-in modules is the [host-store.nix module](src/nix/modules/service/host-store.nix) which performs the bind mounts to make the host Nix store available in the container. # FAQ @@ -101,7 +97,7 @@ Nope, it's just Nix and Docker Compose under the hood. ### Does Arion support Docker images? -Yes, you can still specify a docker image. For example: +Yes, you can also specify a normal Docker image. For example: postgres = { service.image = "postgres:10"; diff --git a/default.nix b/default.nix index 8a370c0..2570b9f 100644 --- a/default.nix +++ b/default.nix @@ -2,4 +2,5 @@ args@{ pkgs ? import ./nix args, ... }: { inherit (pkgs) arion tests; + doc = pkgs.recurseIntoAttrs (import ./doc { inherit pkgs; }); } diff --git a/doc/default.nix b/doc/default.nix new file mode 100644 index 0000000..aedd7ff --- /dev/null +++ b/doc/default.nix @@ -0,0 +1,8 @@ +{ pkgs ? import ../nix {} }: +let + inherit (pkgs) recurseIntoAttrs callPackage; +in + +recurseIntoAttrs { + manual = callPackage ./manual {}; +} diff --git a/doc/manual/.gitignore b/doc/manual/.gitignore new file mode 100644 index 0000000..9ddf569 --- /dev/null +++ b/doc/manual/.gitignore @@ -0,0 +1,4 @@ +introduction.xml +manual.html +options-composition.xml +options-service.xml diff --git a/doc/manual/Makefile b/doc/manual/Makefile new file mode 100644 index 0000000..228afe1 --- /dev/null +++ b/doc/manual/Makefile @@ -0,0 +1,35 @@ +xsltproc = xsltproc --nonet \ + --param section.autolabel 0 \ + --param section.label.includes.component.label 0 \ + --param chapter.autolabel 0 \ + --param chapter.label.includes.component.label 0 \ + --param appendix.autolabel 0 \ + --param appendix.label.includes.component.label 0 \ + --param generate.toc "'book toc,title chapter nop section nop sect1 nop sect2 nop sect3 nop sect4 nop sect5 nop'" \ + --param html.stylesheet \'style.css\' \ + --param xref.with.number.and.title 0 \ + --param toc.section.depth 3 \ + --param admon.style \'\' \ + --param callout.graphics.extension \'.gif\' \ + --param contrib.inline.enabled 0 + +docbookxsl = http://docbook.sourceforge.net/release/xsl/current + +all: manual.html + +manual.html: manual.xml introduction.xml installation.xml options-composition.xml options-service.xml + $(xsltproc) --xinclude --stringparam profile.condition manual \ + $(docbookxsl)/profiling/profile.xsl manual.xml | \ + $(xsltproc) --output manual.html $(docbookxsl)/xhtml/docbook.xsl - + +# -e 's___' -e 's___' +%.xml: %.asciidoc + asciidoctor --backend docbook45 --doctype article $< + sed -e 's///' -i $@ + +options-composition.xml options-service.xml: + echo "options-composition.xml and options-service.xml should be written by the derivation. Are you running in 'nix-shell -A manual'?"; exit 1; fi + +install: all + mkdir -p $(docdir) + cp manual.html style.css $(docdir) diff --git a/doc/manual/default.nix b/doc/manual/default.nix new file mode 100644 index 0000000..afecf52 --- /dev/null +++ b/doc/manual/default.nix @@ -0,0 +1,120 @@ +{ pkgs ? import ../../nix {} +, version ? "none" +# Default sourceUrl is for local development. Works with +# nix-build -A doc.manual +# For release, use: https://github.com/hercules-ci/arion/blob/${version} +, sourceUrl ? "../../.." +}: +let + inherit (pkgs) recurseIntoAttrs callPackage runCommand lib stdenv ; + + nixosManualPath = s: "${pkgs.path}/nixos/doc/manual/${s}"; + + # NixOS module system options in JSON format. + options = { moduleType, description, /*optionsList*/ optionsExpr }: recurseIntoAttrs rec { + optionsXML = + # builtins.toFile "options.xml" (builtins.toXML optionsList); + pkgs.runCommand "options.xml" { + buildInputs = [pkgs.nix pkgs.fakeroot pkgs.jq]; + inherit optionsExpr; + } '' + export NIX_LOG_DIR=$PWD + export NIX_STATE_DIR=$PWD + nix-instantiate --option sandbox false --readonly-mode --eval --expr "$optionsExpr" --xml --strict >$out + ''; + + optionsDocBook = runCommand "options-db.xml" {} '' + optionsXML=${optionsXML} + ${pkgs.buildPackages.libxslt.bin}/bin/xsltproc \ + --stringparam revision '${version}' \ + --stringparam sourceUrl '${sourceUrl}' \ + -o intermediate.xml ${./options-to-docbook.xsl} $optionsXML + ${pkgs.buildPackages.libxslt.bin}/bin/xsltproc \ + -o "$out" ${nixosManualPath "postprocess-option-descriptions.xsl"} intermediate.xml + ''; + }; + +in + +recurseIntoAttrs rec { + compositionOptions = options { + moduleType = "composition"; + description = "List of Arion composition-level options in JSON format"; + optionsExpr = let + src = ../../src/nix; + in '' + let pkgs = import ${pkgs.path} {}; + fixPaths = opt: opt // { + declarations = map (d: "src/nix" + (lib.strings.removePrefix (toString ${src}) (toString d))) opt.declarations; + }; + inherit (pkgs) lib; + composition = import ${src}/eval-composition.nix { inherit pkgs; }; + in map fixPaths (lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList composition.options)) + ''; + }; + serviceOptions = options { + moduleType = "service"; + description = "List of Arion service-level options in JSON format"; + optionsExpr = let + src = ../../src/nix; + in '' + let pkgs = import ${pkgs.path} {}; + fixPaths = opt: opt // { + declarations = map (d: "src/nix" + (lib.strings.removePrefix (toString ${src}) (toString d))) opt.declarations; + }; + inherit (pkgs) lib; + composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; }; + in map fixPaths (lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList composition.options)) + ''; + }; + generatedDocBook = runCommand "generated-docbook" {} '' + mkdir $out + ln -s ${compositionOptions.optionsDocBook} $out/options-composition.xml + ln -s ${serviceOptions.optionsDocBook} $out/options-service.xml + ''; + manual = stdenv.mkDerivation { + src = lib.sourceByRegex ./. [ + "Makefile$" + ".*\.asciidoc$" + ".*\.xsl$" + ".*\.css$" + "^manual.xml$" + "^manual.xml$" + ]; + name = "arion-manual"; + version = version; + buildInputs = [ + (pkgs.libxslt.bin or pkgs.libxslt) + pkgs.asciidoctor + ]; + XML_CATALOG_FILES = "${pkgs.docbook_xsl}/xml/xsl/docbook/catalog.xml"; + inherit generatedDocBook; + configurePhase = '' + export docdir=$out/doc + ''; + postPatch = '' + substituteInPlace manual.xml --subst-var version + ''; + prePatch = '' + cp ${generatedDocBook}/* . + substituteInPlace options-service.xml \ + --replace 'xml:id="appendix-configuration-options"' 'xml:id="appendix-service-options"' \ + --replace 'Configuration Options' 'Service Options' \ + --replace 'xml:id="configuration-variable-list"' 'xml:id="service-variable-list"' \ + ; + substituteInPlace options-composition.xml \ + --replace 'xml:id="appendix-configuration-options"' 'xml:id="appendix-composition-options"' \ + --replace 'Configuration Options' 'Composition Options' \ + --replace 'xml:id="configuration-variable-list"' 'xml:id="composition-variable-list"' \ + ; + ''; + shellHook = '' + live-build() { + patchPhase + inotifywait -e MODIFY -m -r . | while read; do + make + done + } + ''; + }; +} diff --git a/doc/manual/installation.asciidoc b/doc/manual/installation.asciidoc new file mode 100644 index 0000000..6f58112 --- /dev/null +++ b/doc/manual/installation.asciidoc @@ -0,0 +1,37 @@ + += Installation + +== Imperative installation, bleeding edge + +``` +git clone git@github.com:hercules-ci/arion.git +cd arion +nix-env -iA arion -f . +``` + +// TODO: replace the above with something like below +//// + +== Not installing: use it in a project + +TODO: describe: using nix-shell or in a script, building images as + part of nix-build, pinning, see also todomvc-nix. + +== Mac or traditional Linux + +``` +nix-env -iA nixpkgs.arion +``` + +== NixOS + +Add this module to your NixOS configuration: + +``` +{ pkgs, ... } { + virtualisation.docker.enable = true; + environment.systemPackages = [ pkgs.arion ]; +} +``` + +//// diff --git a/doc/manual/introduction.asciidoc b/doc/manual/introduction.asciidoc new file mode 100644 index 0000000..6417cb5 --- /dev/null +++ b/doc/manual/introduction.asciidoc @@ -0,0 +1,33 @@ + += Introduction + +Arion is a tool for building and running applications that +consist of multiple docker containers. It has special support +for docker images that are built with Nix, for a smooth +development experience and improved performance. + +It is built on top of https://docs.docker.com/compose/overview/[Docker +Compose], which implements the container orchestration functionality. + +Instead of configuring the compositions in YAML files like +`docker-compose.yaml`, Arion uses the https://nixos.org/nix/[Nix] +language to declare the compositions. Because of this, Arion gives you +the ability to declare your deployments, configuration and packaging +in the same language. By replacing multiple tools with a single +language, you decrease your mental load and you can more easily +refactor and maintain your configurations. + +Although Arion can be used as a Docker Compose with an improved +configuration front end, there is more to be gained from integrating +with Nix. In particular, the more structured approach of Nix compared +to Dockerfiles allows the following: + + * Build components of your image in _parallel, automatically_. + * Share packages between images, regardless of the order they were + added. + * Improve performance by _skipping_ container + image creation. + * Work with _structured data_ instead of strings, + templates and a multitude of expression languages. + * Refactor across deployments, configuration and + packaging. diff --git a/doc/manual/live-build b/doc/manual/live-build new file mode 100755 index 0000000..860d200 --- /dev/null +++ b/doc/manual/live-build @@ -0,0 +1,3 @@ +#!/usr/bin/env nix-shell +#!nix-shell -A manual +#!nix-shell --run live-build diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml new file mode 100644 index 0000000..237ef2f --- /dev/null +++ b/doc/manual/manual.xml @@ -0,0 +1,25 @@ + + + + + Arion Manual + Version none + + + + Hercules Labs and other Arion contributors + + Author + + + + + + + + + + + diff --git a/doc/manual/options-to-docbook.xsl b/doc/manual/options-to-docbook.xsl new file mode 100644 index 0000000..d6e28c1 --- /dev/null +++ b/doc/manual/options-to-docbook.xsl @@ -0,0 +1,224 @@ + + + + + + + + + + + + + + Configuration Options + + + + + + + + + + + + + + + + + + + + Type: + + + + + (read only) + + + + + + + Default: + + + + + + + + Example: + + + + + + + + + + + + + + + Related packages: + + + + + + + + Declared by: + + + + + + + Defined by: + + + + + + + + + + + + + + + + + + + +'' +'' + + + + + + + + + + null + + + + + + + '''' + + + "" + + + + + + + + + + + + true + + + + + false + + + + + [ + + + + + ] + + + + + + + + + + { + + + = + ; + + } + + + + + (build of ) + + + + + + + + + + / + + + file:// + + + + + + </> + + + + + + + + + + + + + λ + + + + diff --git a/doc/manual/style.css b/doc/manual/style.css new file mode 100644 index 0000000..9b6e154 --- /dev/null +++ b/doc/manual/style.css @@ -0,0 +1,159 @@ + +hr { color: #ddd; margin-top: 3ex; } +h1, h2, h3, h4 { font-weight: bold; } +h1 { font-size: 200%; margin-top: 5ex; } +h2 { font-size: 160%; margin-top: 4ex; } +h3,h4 { font-size: 120%; margin-top: 3ex; } + +/* From Semantic UI */ +body { + font-family: Lato,'Helvetica Neue',Arial,Helvetica,sans-serif; + font-size: 14px; + line-height: 1.4285em; + color: rgba(0,0,0,.87); +} + +code.literal { + background-color: #f7f7f7; +} + +a { + background-color:transparent; + -webkit-text-decoration-skip:objects +} +a { + color:#4183c4; + text-decoration:none +} +a:hover { + color:#1e70bf; + text-decoration:none +} +::-webkit-selection { + background-color:#cce2ff; + color:rgba(0,0,0,.87) +} +::-moz-selection { + background-color:#cce2ff; + color:rgba(0,0,0,.87) +} +::selection { + background-color:#cce2ff; + color:rgba(0,0,0,.87) +} + +/* toc menu */ +@media screen and (min-width: 60em) { + body { + margin-left: 16.5em; + } + div.toc { + position: fixed; + top: 0pt; + left: 1em; + height: 100%; + overflow-y: auto; + width: 15.5em; + } +} +@media screen and (min-width: 90em) { + div.toc { + left: 5em; + } +} +/* hide per chapter toc */ +div.chapter div.toc { + display: none; +} + + +/* From Nixpkgs: */ + +div.book +{ + text-align: center; +} + +div.book > div +{ + /* + * based on https://medium.com/@zkareemz/golden-ratio-62b3b6d4282a + * we do 70 characters per line to fit code listings better + * 70 * (font-size / 1.618) + * expression for emacs: + * (* 70 (/ 1 1.618)) + */ + max-width: 43.2em; + text-align: left; + margin: auto; +} + + +/*************************************************************************** + Special elements: + ***************************************************************************/ + +.term +{ + font-weight: bold; + +} + +div.variablelist dd p, div.glosslist dd p +{ + margin-top: 0em; +} + +div.variablelist dd, div.glosslist dd +{ + margin-left: 1.5em; +} + +div.glosslist dt +{ + font-style: italic; +} + +.varname +{ + color: #004000; +} + +span.command strong +{ + font-weight: normal; + color: #004000; +} + +div.calloutlist table +{ + box-shadow: none; +} + +table +{ + border-collapse: collapse; + box-shadow: 0.4em 0.4em 0.5em #e0e0e0; +} + +table.simplelist +{ + text-align: left; + color: #005aa0; + border: 0; + padding: 5px; + background: #fffff5; + font-weight: normal; + font-style: italic; + box-shadow: none; + margin-bottom: 1em; +} + +div.navheader table, div.navfooter table { + box-shadow: none; +} + +div.affiliation +{ + font-style: italic; +} diff --git a/nix/overlay.nix b/nix/overlay.nix index 7386daa..13ad5dc 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,4 +1,5 @@ self: super: { arion = super.callPackage ../arion.nix {}; tests = super.callPackage ../tests {}; + doc = super.callPackage ../doc {}; } diff --git a/src/nix/eval-service.nix b/src/nix/eval-service.nix new file mode 100644 index 0000000..cf31678 --- /dev/null +++ b/src/nix/eval-service.nix @@ -0,0 +1,25 @@ +{ lib, pkgs, ... }: + +{ modules, host }: +let + composite = lib.evalModules { + check = true; + modules = builtinModules ++ modules; + }; + + builtinModules = [ + argsModule + ./modules/service/docker-compose-service.nix + ./modules/service/host-store.nix + ./modules/service/host.nix + ]; + + argsModule = { + _file = ./docker-compose.nix; + key = ./docker-compose.nix; + config._module.args.pkgs = lib.mkForce pkgs; + config.host = host; + }; + +in + composite diff --git a/src/nix/modules/composition/docker-compose.nix b/src/nix/modules/composition/docker-compose.nix index 63aeace..8ce8393 100644 --- a/src/nix/modules/composition/docker-compose.nix +++ b/src/nix/modules/composition/docker-compose.nix @@ -10,47 +10,28 @@ */ { pkgs, lib, config, ... }: - let - evalService = name: modules: - let - composite = lib.evalModules { - check = true; - modules = builtinModules ++ modules; - }; - - builtinModules = [ - argsModule - ../service/docker-compose-service.nix - ../service/host-store.nix - ../service/host.nix - ]; - - argsModule = { - _file = ./docker-compose.nix; - key = ./docker-compose.nix; - config._module.args.pkgs = lib.mkForce pkgs; - config.host = config.host; - }; - - in - composite.config.build.service; +evalService = name: modules: (pkgs.callPackage ../../eval-service.nix {} { inherit modules; inherit (config) host; }).config.build.service; in { options = { build.dockerComposeYaml = lib.mkOption { type = lib.types.package; + description = "A derivation that produces a docker-compose.yaml file for this composition."; }; build.dockerComposeYamlText = lib.mkOption { type = lib.types.string; + description = "The text of build.dockerComposeYaml."; }; docker-compose.raw = lib.mkOption { type = lib.types.attrs; + description = "Nested attribute set that will be turned into the docker-compose.yaml file, using Nix's toJSON builtin."; }; docker-compose.services = lib.mkOption { default = {}; type = with lib.types; attrsOf (coercedTo unspecified (a: [a]) (listOf unspecified)); + description = "A attribute set of service configurations. A service specifies how to run an image. Each of these service configurations is specified using modules whose options are described in the Service Options section."; }; }; config = { diff --git a/src/nix/modules/composition/host-environment.nix b/src/nix/modules/composition/host-environment.nix index 16bf23e..3e36531 100644 --- a/src/nix/modules/composition/host-environment.nix +++ b/src/nix/modules/composition/host-environment.nix @@ -8,8 +8,9 @@ description = '' The numeric user id (UID) of the user running arion. - Assuming this user id is helpful when dealing with the user's - files, mounted into the container as a volume. + This lets you to write modules that interact with the host + user's files, which is helpful for local development, but not + intended for production-like deployment scenarios. ''; }; diff --git a/src/nix/modules/service/docker-compose-service.nix b/src/nix/modules/service/docker-compose-service.nix index ef15a81..8e63de6 100644 --- a/src/nix/modules/service/docker-compose-service.nix +++ b/src/nix/modules/service/docker-compose-service.nix @@ -9,79 +9,116 @@ let inherit (lib) mkOption types; inherit (types) listOf nullOr attrsOf string either int bool; + + dockerComposeRef = fragment: + ''See Docker Compose#${fragment}''; + dockerComposeKitchenSink = '' + Analogous to the docker run counterpart. + + ${dockerComposeRef "domainname-hostname-ipc-mac_address-privileged-read_only-shm_size-stdin_open-tty-user-working_dir"} + ''; in { options = { service.volumes = mkOption { type = listOf types.unspecified; default = []; + description = ""; }; service.build.context = mkOption { type = nullOr string; default = null; + description = '' + Locates a Dockerfile to use for creating an image to use in this service. + + ${dockerComposeRef "context"} + ''; }; service.hostname = mkOption { type = nullOr string; default = null; + description = dockerComposeKitchenSink; }; service.environment = mkOption { type = attrsOf (either string int); default = {}; + description = dockerComposeRef "environment"; }; service.image = mkOption { type = string; + description = dockerComposeRef "image"; }; service.command = mkOption { type = nullOr types.unspecified; default = null; + description = dockerComposeRef "command"; }; service.depends_on = mkOption { type = listOf string; default = []; + description = dockerComposeRef "depends_on"; }; service.links = mkOption { type = listOf string; default = []; + description = dockerComposeRef "links"; }; service.external_links = mkOption { type = listOf string; default = []; + description = dockerComposeRef "external_links"; }; service.extra_hosts = mkOption { type = listOf string; default = []; + description = dockerComposeRef "extra_hosts"; }; service.working_dir = mkOption { type = nullOr string; default = null; + description = dockerComposeKitchenSink; }; service.privileged = mkOption { type = nullOr bool; default = null; + description = dockerComposeKitchenSink; }; service.entrypoint = mkOption { type = nullOr string; default = null; + description = dockerComposeRef "entrypoint"; }; service.restart = mkOption { type = nullOr string; default = null; + description = dockerComposeRef "restart"; }; service.ports = mkOption { type = listOf types.unspecified; default = []; description = '' Expose ports on host. "host:container" or structured. - See https://docs.docker.com/compose/compose-file/#ports + + ${dockerComposeRef "ports"} ''; }; service.expose = mkOption { type = listOf string; default = []; + description = dockerComposeRef "expose"; }; build.service = mkOption { type = attrsOf types.unspecified; + description = '' + Raw input for the service in docker-compose.yaml. + + You should not need to use this option. If anything is + missing, please contribute the missing option. + + This option is user accessible because it may serve as an + escape hatch for some. + ''; }; };