Merge pull request #60 from hercules-ci/arion-hs

Arion hs

.envrc

@@ -0,0 +1,5 @@
+eval "$(lorri direnv)"
+
+# Use system PKI
+unset SSL_CERT_FILE
+unset NIX_SSL_CERT_FILE

.gitignore

@@ -1,2 +1,7 @@
 result
 result-*
+
+dist/
+dist-newstyle/
+cabal.project.local
+

CHANGELOG.md

@@ -0,0 +1,6 @@
+# Revision history for arion-compose
+
+## 0.1.0.0 -- YYYY-mm-dd
+
+* First version. Released on an unsuspecting world.
+* *BREAKING:* useHostStore now uses a proper empty base image, like `scratch`.

Setup.hs

@@ -0,0 +1,2 @@
+import Distribution.Simple
+main = defaultMain

arion-compose.cabal

@@ -0,0 +1,81 @@
+cabal-version:       2.4
+
+name:                arion-compose
+version:             0.1.0.0
+synopsis:            Run docker-compose with help from Nix/NixOS
+-- description:
+homepage:            https://github.com/hercules-ci/arion#readme
+-- bug-reports:
+license:             Apache-2.0
+license-file:        LICENSE
+author:              Robert Hensing
+maintainer:          robert@hercules-ci.com
+-- copyright:
+-- category:
+extra-source-files:  CHANGELOG.md, README.asciidoc
+write-ghc-enviroment-files:
+                     never
+data-files:          nix/*.nix
+                   , nix/modules/composition/*.nix
+                   , nix/modules/nixos/*.nix
+                   , nix/modules/service/*.nix
+
+-- all data is verbatim from some sources
+data-dir:            src
+
+common deps
+  build-depends:     base ^>=4.12.0.0
+                   , aeson
+                   , aeson-pretty
+                   , async
+                   , bytestring
+                   , directory
+                   , lens
+                   , lens-aeson
+                   , process
+                   , temporary
+                   , text
+                   , protolude
+                   , unix
+
+flag ghci
+  default: False
+  manual: True
+
+library
+  import:              deps
+  exposed-modules:     Arion.Nix
+                       Arion.Aeson
+                       Arion.DockerCompose
+                       Arion.Images
+                       Arion.Services
+  other-modules:       Paths_arion_compose
+  -- other-extensions:
+  hs-source-dirs:      src/haskell/lib
+  default-language:    Haskell2010
+
+executable arion
+  import:              deps
+  main-is:             Main.hs
+  -- other-modules:
+  -- other-extensions:
+  build-depends:       optparse-applicative
+                     , arion-compose
+  hs-source-dirs:      src/haskell/exe
+  default-language:    Haskell2010
+
+test-suite arion-unit-tests
+  import:              deps
+  if flag(ghci)
+    hs-source-dirs:      src/haskell/lib
+    ghc-options:       -Wno-missing-home-modules
+  type:                exitcode-stdio-1.0
+  main-is:             TestMain.hs
+  other-modules:       Spec
+                     , Arion.NixSpec
+  -- other-extensions:
+  build-depends:       arion-compose
+                     , hspec
+                     , QuickCheck
+  hs-source-dirs:      src/haskell/test
+  default-language:    Haskell2010

arion.nix

@@ -1,38 +0,0 @@
-{ stdenv, lib
-, coreutils, docker_compose, jq
-}:
-let
-
-  arion = stdenv.mkDerivation {
-    name = "arion";
-    src = ./src;
-    unpackPhase = "";
-    buildPhase = "";
-    installPhase = ''
-      mkdir -p $out/bin $out/share/arion
-      cp -a nix $out/share/arion/
-      cp -a arion-image $out/share/arion/
-      tar -czf $out/share/arion/arion-image/tarball.tar.gz -C arion-image/tarball .
-      substitute arion $out/bin/arion \
-        --subst-var-by path ${lib.makeBinPath [jq coreutils docker_compose]} \
-        --subst-var-by nix_dir $out/share/arion/nix \
-        ;
-      chmod a+x $out/bin/arion
-    '';
-    inherit passthru;
-  };
-
-  passthru = {
-    inherit eval build;
-  };
-
-  eval = import "${nix_dir}/eval-composition.nix";
-
-  build = args@{...}:
-    let composition = eval args;
-    in composition.config.build.dockerComposeYaml;
-
-  nix_dir = "${arion.outPath}/share/arion/nix";
-
-in
-  arion

build

@@ -0,0 +1,6 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash
+
+# Build the Haskell package via cabal, outside Nix
+
+cabal new-build --write-ghc-environment-files=never

cabal.project

@@ -0,0 +1 @@
+packages: .

default.nix

@@ -1,6 +1,6 @@
-args@{ pkgs ? import ./nix args, ... }:
+{ pkgs ? import ./nix {}
-
+, haskellPackages ? pkgs.haskellPackages
+}:
 {
-  inherit (pkgs) arion tests;
+  arion = import ./nix/arion.nix { inherit pkgs haskellPackages; };
-  doc = pkgs.recurseIntoAttrs (import ./doc { inherit pkgs; });
 }

doc/manual/default.nix

@@ -69,7 +69,7 @@ let
             declarations = map (d: "src/nix" + (lib.strings.removePrefix (toString ${src}) (toString d))) opt.declarations;
           };
           inherit (pkgs) lib;
-          composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; name = abort "The manual's service options section must not depend on the service name."; };
+          composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; name = abort "The manual's service options section must not depend on the service name."; composition = abort "The manual's service options must not depend on the composition."; };
       in map fixPaths (lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList composition.options))
     '';
   };

examples/full-nixos/arion-pkgs.nix

@@ -1,2 +1,6 @@
 # Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH
-import <nixpkgs> {}
+import <nixpkgs> {
+  # We specify the architecture explicitly. Use a Linux remote builder when
+  # calling arion from other platforms.
+  system = "x86_64-linux";
+}

examples/minimal/arion-pkgs.nix

@@ -1,2 +1,6 @@
 # Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH
-import <nixpkgs> {}
+import <nixpkgs> {
+  # We specify the architecture explicitly. Use a Linux remote builder when
+  # calling arion from other platforms.
+  system = "x86_64-linux";
+}

examples/nixos-unit/arion-pkgs.nix

@@ -1,2 +1,6 @@
 # Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH
-import <nixpkgs> {}
+import <nixpkgs> {
+  # We specify the architecture explicitly. Use a Linux remote builder when
+  # calling arion from other platforms.
+  system = "x86_64-linux";
+}

live-check

@@ -0,0 +1,12 @@
+#!/usr/bin/env nix-shell
+#!nix-shell ./shell.nix
+#!nix-shell -i bash
+set -eux -o pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+ghcid \
+  --command 'ghci -isrc/haskell/exe src/haskell/exe/Main.hs' \
+  --reload=src/haskell \
+  --restart=arion-compose.cabal \
+  ;

live-unit-tests

@@ -0,0 +1,13 @@
+#!/usr/bin/env nix-shell
+#!nix-shell ./shell.nix
+#!nix-shell -i bash
+set -eux -o pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+ghcid \
+  --command 'cabal v2-repl arion-compose:arion-unit-tests --flags ghci --write-ghc-environment-files=never' \
+  --test=Main.main \
+  --reload=src/haskell \
+  --restart=arion-compose.cabal \
+  ;

nix/arion.nix

@@ -0,0 +1,42 @@
+{ pkgs ? import ./. {}
+, lib ? pkgs.lib
+, haskellPackages ? pkgs.haskellPackages
+, arion-compose ? import ./haskell-arion-compose.nix { inherit pkgs haskellPackages; }
+}:
+
+let
+  inherit (pkgs.haskell.lib) justStaticExecutables overrideCabal;
+
+  srcDir = ../src;
+  eval = import (srcDir + "/nix/eval-composition.nix");
+  build = args@{...}:
+    let composition = eval args;
+    in composition.config.build.dockerComposeYaml;
+
+in
+  justStaticExecutables (overrideCabal arion-compose (o: {
+    buildTools = o.buildTools ++ [pkgs.makeWrapper];
+    passthru = o.passthru // {
+      inherit eval build;
+    };
+    pname = "arion"; # Cover up the needlessly long Haskell package name
+
+    # PYTHONPATH
+    #
+    # We close off the python module search path!
+    #
+    # Accepting directories from the environment into the search path
+    # tends to break things. Docker Compose does not have a plugin
+    # system as far as I can tell, so I don't expect this to break a
+    # feature, but rather to make the program more robustly self-
+    # contained.
+
+    postInstall = ''${o.postInstall or ""}
+      mkdir -p $out/libexec
+      mv $out/bin/arion $out/libexec
+      makeWrapper $out/libexec/arion $out/bin/arion \
+        --unset PYTHONPATH \
+        --prefix PATH : ${lib.makeBinPath [ pkgs.docker-compose ]} \
+        ;
+    '';
+  }))

nix/ci.nix

@@ -0,0 +1,6 @@
+args@{ pkgs ? import ./default.nix args, system ? null, ... }:
+
+{
+  inherit (pkgs) arion tests;
+  doc = pkgs.recurseIntoAttrs (import ../doc { inherit pkgs; });
+}

nix/default.nix

@@ -1,8 +1,12 @@
 /**
  * This is the entry-point for all nix execution in this project.
  */
-{ nixpkgsSrc ? ./nixpkgs.nix, ... }:
+{ nixpkgsSrc ? ./nixpkgs.nix
-import (import ./nixpkgs.nix) {
+, system ? null
+, ...
+}:
+
+import (import ./nixpkgs.nix) ({
   # Makes the config pure as well. See <nixpkgs>/top-level/impure.nix:
   config = {
   };
@@ -10,4 +14,6 @@ import (import ./nixpkgs.nix) {
     # all the packages are defined there:
     (import ./overlay.nix)
   ];
-}
+} // (if system == null then {} else {
+  inherit system;
+}))

nix/haskell-arion-compose.nix

@@ -0,0 +1,14 @@
+
+# NOTE: This file produces a haskell library, not the arion package!
+
+{ pkgs ? import ./default.nix {}, haskellPackages ? pkgs.haskellPackages }:
+let
+  inherit (pkgs.haskell.lib) overrideCabal addBuildTools;
+in
+  overrideCabal (addBuildTools (haskellPackages.callCabal2nix "arion-compose" ./.. {}) [pkgs.nix]) (o: o // {
+    preCheck = ''
+      export NIX_LOG_DIR=$TMPDIR
+      export NIX_STATE_DIR=$TMPDIR
+      export NIX_PATH=nixpkgs=${pkgs.path}
+    '';
+  })

nix/haskell-overlay.nix

@@ -0,0 +1,4 @@
+self: super: hself: hsuper:
+{
+  arion-compose = import ./haskell-arion-compose.nix { pkgs = self; haskellPackages = hself; };
+}

nix/nixpkgs.nix

@@ -1,5 +1,5 @@
 # to update: $ nix-prefetch-url --unpack url
 builtins.fetchTarball {
-  url = "https://github.com/NixOS/nixpkgs/archive/be445a9074f139d63e704fa82610d25456562c3d.tar.gz";
+  url = "https://github.com/NixOS/nixpkgs/archive/bd5e8f35c2e9d1ddc9cd2fea7a23563336d54acb.tar.gz";
-  sha256 = "15dc7gdspimavcwyw9nif4s59v79gk18rwsafylffs9m1ld2dxwa";
+  sha256 = "1wnzqqijrwf797nb234q10zb1h7086njradkkrx3a15b303grsw4";
 }

nix/overlay.nix

@@ -1,5 +1,25 @@
-self: super: {
+self: super:
-  arion = super.callPackage ../arion.nix {};
+let
+  inherit (self.arion-project) haskellPkgs;
+  inherit (super) lib;
+
+in
+{
+
+  arion = import ./arion.nix { pkgs = self; };
   tests = super.callPackage ../tests {};
   doc = super.callPackage ../doc {};
+
+  arion-project = super.recurseIntoAttrs {
+    haskellPkgs = super.haskellPackages.extend (import ./haskell-overlay.nix self super);
+    shell = haskellPkgs.shellFor {
+      packages = p: [p.arion-compose];
+      buildInputs = [
+        haskellPkgs.cabal-install
+        haskellPkgs.ghcid
+        super.docker-compose
+        (import ~/h/ghcide-nix {}).ghcide-ghc864
+      ];
+    };
+  };
 }

repl

@@ -0,0 +1,6 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash
+
+# A Haskell REPL for hacking on the Haskell code
+
+cabal new-repl --write-ghc-environment-files=never

run-arion

@@ -0,0 +1,15 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash
+#!nix-shell ./shell.nix
+
+# For quick manual testing of a hacked arion
+
+# NB: Only works inside the project directory
+
+cabal \
+  new-run \
+  --write-ghc-environment-files=never \
+  :pkg:arion-compose:exe:arion \
+  -- \
+  "$@" \
+  ;

run-arion-via-nix

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# For manual testing of a hacked arion built via Nix.
+# Works when called from outside the project directory.
+
+exec nix run -f "$(dirname ${BASH_SOURCE[0]})" arion -c arion "$@"

shell.nix

@@ -0,0 +1 @@
+args@{...}: (import ./nix args).arion-project.shell

src/arion

@@ -1,317 +0,0 @@
-#!/usr/bin/env bash
-
-# Close off the python module search path
-#
-# Accepting directories from the environment into the search path
-# tends to break things. Docker Compose does not have a plugin
-# system as far as I can tell, so I don't expect this to break a
-# feature, but rather to make the program more robustly self-
-# contained.
-unset PYTHONPATH
-
-set -euo pipefail
-
-export PATH="@path@:$PATH"
-
-nix_dir="@nix_dir@"
-docker_compose_args=()
-files=()
-command="docker-compose"
-pkgs_argument="./arion-pkgs.nix"
-
-debug() {
-    # echo "$@"
-    :
-}
-
-while test $# != 0; do
-    case "$1" in
-        -f|--file)
-            shift
-            files+=("$1")
-            ;;
-        -f*)
-            files+=("${1/#-f/}")
-            ;;
-        --file=*)
-            files+=("${1/#--file=}")
-            ;;
-        --pkgs)
-            shift
-            pkgs_argument="$1"
-            ;;
-        -h|--help|help)
-            command="help"
-            shift
-            break
-            ;;
-        cat)
-            command="$1"
-            shift
-            break
-            ;;
-        repl)
-            command="$1"
-            shift
-            break
-            ;;
-        exec)
-            command="$1"
-            shift
-            break
-            ;;
-        docker-compose)
-            command="docker-compose"
-            shift
-            break
-            ;;
-        *)
-            break
-            ;;
-    esac
-    shift
-done
-
-while test $# != 0; do
-    docker_compose_args+=("$1")
-    shift
-done
-
-case "$command" in
-    help)
-        cat <<EOF
-
-Arion wraps your system's docker-compose, providing a NixOps-like
-experience for simple container deployments.
-
-Usage:
-  arion up|logs|...     - execute docker-compose commands
-  arion cat             - display raw docker-compose.yaml
-  arion config          - validate and display the config file
-  arion repl            - explore the config interactively
-  arion help
-  arion docker-compose help
-  arion docker-compose help up|logs|...
-
-Top-level arion options
-
-  These must be provided before the command.
-
-  --file FILE       Use FILE instead of the default ./arion-compose.nix
-                    Can be specified multiple times for a merged configuration.
-  --pkgs EXPR       Use EXPR instead of ./arion-pkgs.nix to get the
-                    Nixpkgs attrset used for bootstrapping and evaluating
-                    the configuration.
-
-EOF
-        exit 0
-        ;;
-    *)
-        ;;
-esac
-
-if [[ ${#files[@]} == 0 ]]; then
-    files=("./arion-compose.nix")
-fi
-
-
-debug docker_compose_args: "${docker_compose_args[@]}"
-debug files: "${files[@]}"
-
-docker_compose_yaml=.tmp-nix-docker-compose-$$-$RANDOM.yaml
-cleanup() {
-  rm -f $docker_compose_yaml
-}
-trap cleanup EXIT
-
-modules="["
-
-for file in "${files[@]}"; do
-    case "$file" in
-        /*)
-            modules="$modules (/. + $(printf '"%q"' "$file"))"
-            ;;
-        *)
-            modules="$modules (./. + $(printf '"/%q"' "$file"))"
-            ;;
-    esac
-done
-
-modules="$modules ]"
-
-debug modules: "$modules"
-
-old_IFS="$IFS"
-IFS=""
-args=(
-     )
-IFS="$old_IFS"
-for arg in "${args[@]}"; do
-    echo "arg: $arg"
-done
-
-do_eval() {
-  echo 1>&2 "Evaluating configuration..."
-  # read-write-mode is required for import from derivation
-  nix-instantiate \
-    "$nix_dir/eval-composition.nix" \
-    --eval \
-    --read-write-mode \
-    --json \
-    --argstr uid "$UID" \
-    --arg modules "$modules" \
-    --arg pkgs "$pkgs_argument" \
-    --show-trace \
-    --attr 'config.build.dockerComposeYamlText' \
-    | jq -r . >$docker_compose_yaml;
-}
-
-do_build() {
-  echo 1>&2 "Building configuration..."
-  nix-build \
-    "$nix_dir/eval-composition.nix" \
-    --out-link $docker_compose_yaml \
-    --argstr uid "$UID" \
-    --arg modules "$modules" \
-    --arg pkgs "$pkgs_argument" \
-    --show-trace \
-    --attr 'config.build.dockerComposeYaml' \
-    >/dev/null ;
-
-  echo 1>&2 "Ensuring required images are loaded..."
-  jq -r <"$docker_compose_yaml" \
-    '.["x-arion"].images | map("  - " + .imageName + ":" + .imageTag) | join("\n")'
-  eval "$(
-    jq -r '.["docker-compose"]["x-arion"].images as $images
-         | .["existing-images"] as $loaded
-         | $images
-         | map(
-             if $loaded[.imageName + ":" + .imageTag]
-             then ""
-             else "docker load <" + .image + ";" end
-           )
-         | join("\n")
-     ' <<EOF
-{
-  "docker-compose": $(cat $docker_compose_yaml),
-  "existing-images": {
-    $(docker images \
-        --filter "dangling=false" \
-        --format '"{{.Repository}}:{{.Tag}}": true,')
-    "": false
-  }
-}
-EOF
-      )"
-}
-
-
-do_repl() {
-    # nix repl doesn't autocall its <FILES> arguments
-    # so we improvise. We need a file in this directory
-    # to make sure that all paths are as expected :(
-    trap do_repl_cleanup EXIT;
-
-    REPL_TMP=.tmp-repl-$$-$RANDOM
-    cat <<EOF
-Launch a repl for you, using a temporary file: $REPL_TMP.
-
-This loads the configuration from the modules
-  ${files[*]}
-
-To get started:
-
-To see deployment-wide configuration
-  type config. and hit TAB
-To see the services
-  type config.docker-compose.evaluatedServices TAB or ENTER
-To bring the top-level Nixpkgs attributes into scope
-  type :a (config._module.args.pkgs) // { inherit config; }
-
-EOF
-    cat >"$REPL_TMP" <<EOF
-import $nix_dir/eval-composition.nix {
-  uid = "$UID";
-  modules = $modules;
-  pkgs = $pkgs_argument;
-}
-EOF
-
-    nix repl \
-        "$REPL_TMP" \
-        ;
-}
-do_repl_cleanup() {
-    rm -f $REPL_TMP
-}
-
-run_exec() {
-    case "${#docker_compose_args[@]}" in
-        0)
-            echo "As an argument to exec, please specify a service"
-            exit 1
-            ;;
-        1)
-            case "${docker_compose_args[0]}" in
-                -*|--*)
-                    echo "As an argument to exec, please specify a service"
-                    echo "Note that executing the default command currently does not support"
-                    echo "docker-compose options."
-                    # This requires parsing the options, in order to figure out
-                    # which service to invoke.
-                    exit 1
-                    ;;
-                *)
-                    serviceName="${docker_compose_args[0]}"
-                    do_eval
-                    default_args=()
-                    while read arg; do
-                        default_args+=("$arg")
-                    done < <(
-                        jq < "$docker_compose_yaml" \
-                            --arg serviceName "$serviceName" \
-                            -r \
-                            '.["x-arion"].serviceInfo[$serviceName].defaultExec | tostream | .[1] | select(.)'
-                        )
-                    docker-compose -f $docker_compose_yaml exec "$serviceName" "${default_args[@]}"
-                    ;;
-            esac
-            ;;
-        *)
-            do_eval
-            docker-compose -f $docker_compose_yaml exec "${docker_compose_args[@]}"
-            ;;
-    esac
-}
-
-case "$command" in
-    cat)
-        do_eval
-        jq . < "$docker_compose_yaml"
-        ;;
-    repl)
-        do_repl
-        ;;
-    exec)
-        run_exec "$@"
-        ;;
-    docker-compose)
-        if [[ ${#docker_compose_args[@]} != 0
-           && ${docker_compose_args[0]} != "help"
-           && ${docker_compose_args[0]} != "version"
-            ]]; then
-            case "${docker_compose_args[0]}" in
-                help|version)
-                    :
-                    ;;
-                config|down|events|exec|images|kill|logs|pause|port|ps|rm|stop|top|unpause)
-                    do_eval
-                    ;;
-                *)
-                    do_build
-                    ;;
-            esac
-        fi
-        docker-compose -f $docker_compose_yaml "${docker_compose_args[@]}"
-        ;;
-esac

src/arion-image/Dockerfile

@@ -1,3 +1,5 @@
 FROM scratch
-COPY passwd /etc/passwd
+# scratch itself can't be run.
-ADD tarball.tar.gz /
+
+# This seems like a no-op:
+CMD []

src/arion-image/passwd

@@ -1,2 +0,0 @@
-root:x:0:0:System administrator:/root:/bin/sh
-nobody:x:65534:65534:Unprivileged account (don't use!):/var/empty:/run/current-system/sw/bin/nologin

src/arion-image/tarball/bin/sh

@@ -1 +0,0 @@
-/run/system/bin/sh

src/arion-image/tarball/usr/bin/env

@@ -1 +0,0 @@
-/run/system/usr/bin/env

src/haskell/exe/Main.hs

@@ -0,0 +1,273 @@
+{-# LANGUAGE NoImplicitPrelude #-}
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE ApplicativeDo #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+import Protolude hiding (Down, option)
+
+import           Arion.Nix
+import           Arion.Aeson
+import           Arion.Images (loadImages)
+import qualified Arion.DockerCompose as DockerCompose
+import           Arion.Services (getDefaultExec)
+
+import Options.Applicative
+import Control.Applicative
+import Control.Monad.Fail
+
+import qualified Data.Aeson.Encode.Pretty
+import qualified Data.Text as T
+import qualified Data.Text.IO as T
+import qualified Data.Text.Lazy as TL
+import qualified Data.Text.Lazy.Builder as TB
+
+import qualified Data.List.NonEmpty as NE
+import Data.List.NonEmpty (NonEmpty(..))
+
+import Control.Arrow ((>>>))
+
+import System.Posix.User (getRealUserID)
+
+data CommonOptions =
+  CommonOptions
+    { files :: NonEmpty FilePath
+    , pkgs :: Text
+    , nixArgs :: [Text]
+    }
+  deriving (Show)
+
+newtype DockerComposeArgs =
+  DockerComposeArgs { unDockerComposeArgs :: [Text] }
+
+ensureConfigFile :: [FilePath] -> NonEmpty FilePath
+ensureConfigFile []     = "./arion-compose.nix" :| []
+ensureConfigFile (x:xs) = x :| xs
+
+parseOptions :: Parser CommonOptions
+parseOptions = do
+    files <-
+      ensureConfigFile <$>
+        many (strOption
+               (  short 'f'
+               <> long "file"
+               <> metavar "FILE"
+               <> help "Use FILE instead of the default ./arion-compose.nix. \
+                        \Can be specified multiple times for a merged configuration" ))
+    pkgs <- T.pack <$> strOption
+          (  short 'p'
+          <> long "pkgs"
+          <> metavar "EXPR"
+          <> showDefault
+          <> value "./arion-pkgs.nix"
+          <> help "Use Nix expression EXPR to get the Nixpkgs attrset used for bootstrapping \
+                   \and evaluating the configuration." )
+    showTrace <- flag False True (long "show-trace"
+                    <> help "Causes Nix to print out a stack trace in case of Nix expression evaluation errors.")
+    -- TODO --option support (https://github.com/pcapriotti/optparse-applicative/issues/284)
+    userNixArgs <- many (T.pack <$> strOption (long "nix-arg" <> metavar "ARG" <> help "Pass an extra argument to nix. Example: --nix-arg --option --nix-arg substitute --nix-arg false"))
+    pure $
+      let nixArgs = userNixArgs <|> "--show-trace" <$ guard showTrace
+      in CommonOptions{..}
+
+textArgument = fmap T.pack . strArgument
+
+parseCommand :: Parser (CommonOptions -> IO ())
+parseCommand =
+  hsubparser
+    (    command "cat" (info (pure runCat) (progDesc "Spit out the docker compose file as JSON" <> fullDesc))
+      <> command "repl" (info (pure runRepl) (progDesc "Start a nix repl for the whole composition" <> fullDesc))
+      <> command "exec" (info (parseExecCommand) (progDesc "Execute a command in a running container" <> fullDesc))
+    )
+  <|>
+  hsubparser
+    (    commandDC runBuildAndDC "build" "Build or rebuild services"
+      <> commandDC runBuildAndDC "bundle" "Generate a Docker bundle from the Compose file"
+      <> commandDC runEvalAndDC "config" "Validate and view the Compose file"
+      <> commandDC runBuildAndDC "create" "Create services"
+      <> commandDC runEvalAndDC "down" "Stop and remove containers, networks, images, and volumes"
+      <> commandDC runEvalAndDC "events" "Receive real time events from containers"
+      <> commandDC runDC "help" "Get help on a command"
+      <> commandDC runEvalAndDC "images" "List images"
+      <> commandDC runEvalAndDC "kill" "Kill containers"
+      <> commandDC runEvalAndDC "logs" "View output from containers"
+      <> commandDC runEvalAndDC "pause" "Pause services"
+      <> commandDC runEvalAndDC "port" "Print the public port for a port binding"
+      <> commandDC runEvalAndDC "ps" "List containers"
+      <> commandDC runBuildAndDC "pull" "Pull service images"
+      <> commandDC runBuildAndDC "push" "Push service images"
+      <> commandDC runBuildAndDC "restart" "Restart services"
+      <> commandDC runEvalAndDC "rm" "Remove stopped containers"
+      <> commandDC runBuildAndDC "run" "Run a one-off command"
+      <> commandDC runBuildAndDC "scale" "Set number of containers for a service"
+      <> commandDC runBuildAndDC "start" "Start services"
+      <> commandDC runEvalAndDC "stop" "Stop services"
+      <> commandDC runEvalAndDC "top" "Display the running processes"
+      <> commandDC runEvalAndDC "unpause" "Unpause services"
+      <> commandDC runBuildAndDC "up" "Create and start containers"
+      <> commandDC runDC "version" "Show the Docker-Compose version information"
+
+      <> metavar "DOCKER-COMPOSE-COMMAND"
+      <> commandGroup "Docker Compose Commands:"
+    )
+
+parseAll :: Parser (IO ())
+parseAll =
+  flip ($) <$> parseOptions <*> parseCommand
+
+parseDockerComposeArgs :: Parser DockerComposeArgs
+parseDockerComposeArgs =
+  DockerComposeArgs <$>
+    many (argument (T.pack <$> str) (metavar "DOCKER-COMPOSE ARGS..."))
+
+commandDC
+  :: (Text -> DockerComposeArgs -> CommonOptions -> IO ())
+  -> Text
+  -> Text
+  -> Mod CommandFields (CommonOptions -> IO ())
+commandDC run cmdStr help =
+  command
+    (T.unpack cmdStr)
+    (info
+      (run cmdStr <$> parseDockerComposeArgs)
+      (progDesc (T.unpack help) <> fullDesc <> forwardOptions))
+
+
+--------------------------------------------------------------------------------
+
+runDC :: Text -> DockerComposeArgs -> CommonOptions -> IO ()
+runDC cmd (DockerComposeArgs args) opts = do
+  DockerCompose.run DockerCompose.Args
+    { files = []
+    , otherArgs = [cmd] ++ args
+    }
+
+runBuildAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO ()
+runBuildAndDC cmd dopts opts = do
+  ea <- defaultEvaluationArgs opts
+  Arion.Nix.withBuiltComposition ea $ \path -> do
+    loadImages path
+    DockerCompose.run DockerCompose.Args
+      { files = [path]
+      , otherArgs = [cmd] ++ unDockerComposeArgs dopts
+      }
+
+runEvalAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO ()
+runEvalAndDC cmd dopts opts = do
+  ea <- defaultEvaluationArgs opts
+  Arion.Nix.withEvaluatedComposition ea $ \path ->
+    DockerCompose.run DockerCompose.Args
+      { files = [path]
+      , otherArgs = [cmd] ++ unDockerComposeArgs dopts
+      }
+
+defaultEvaluationArgs :: CommonOptions -> IO EvaluationArgs
+defaultEvaluationArgs co = do
+  uid <- getRealUserID
+  pure EvaluationArgs
+    { evalUid = fromIntegral uid
+    , evalModules = files co
+    , evalPkgs = pkgs co
+    , evalWorkDir = Nothing
+    , evalMode = ReadWrite
+    , evalUserArgs = nixArgs co
+    }
+
+runCat :: CommonOptions -> IO ()
+runCat co = do
+  v <- Arion.Nix.evaluateComposition =<< defaultEvaluationArgs co
+  T.hPutStrLn stdout (pretty v)
+
+runRepl :: CommonOptions -> IO ()
+runRepl co = do
+  putErrText
+    "Launching a repl for you. To get started:\n\
+    \\n\
+    \To see deployment-wide configuration\n\
+    \  type config. and hit TAB\n\
+    \To see the services\n\
+    \  type config.docker-compose.evaluatedServices TAB or ENTER\n\
+    \To bring the top-level Nixpkgs attributes into scope\n\
+    \  type :a (config._module.args.pkgs) // { inherit config; }\n\
+    \"
+  Arion.Nix.replForComposition =<< defaultEvaluationArgs co
+
+detachFlag :: Parser Bool
+detachFlag = flag False True (long "detach" <> short 'd' <> help "Detached mode: Run command in the background.")
+
+privilegedFlag :: Parser Bool
+privilegedFlag = flag False True (long "privileged" <> help "Give extended privileges to the process.")
+
+userOption :: Parser Text
+userOption = strOption (long "user" <> short 'u' <> help "Run the command as this user.")
+
+noTTYFlag :: Parser Bool
+noTTYFlag = flag False True (short 'T' <> help "Disable pseudo-tty allocation. By default `exec` allocates a TTY.")
+
+indexOption :: Parser Int
+indexOption = option
+  (auto >>= \i -> i <$ unless (i >= 1) (fail "container index must be >= 1"))
+  (long "index" <> value 1 <> help "Index of the container if there are multiple instances of a service.")
+
+envOption :: Parser (Text, Text)
+envOption = option (auto >>= spl) (long "env" <> short 'e' <> help "Set environment variables (can be used multiple times, not supported in Docker API < 1.25)")
+  where spl s = case T.break (== '=') s of
+          (_, "") -> fail "--env parameter needs to combine key and value with = sign"
+          (k, ev) -> pure (k, T.drop 1 ev)
+
+workdirOption :: Parser Text
+workdirOption = strOption (long "workdir" <> short 'w' <> metavar "DIR" <> help "Working directory in which to start the command in the container.")
+
+parseExecCommand :: Parser (CommonOptions -> IO ())
+parseExecCommand = runExec
+  <$> detachFlag
+  <*> privilegedFlag
+  <*> optional userOption
+  <*> noTTYFlag
+  <*> indexOption
+  <*> many envOption
+  <*> optional workdirOption
+  <*> textArgument (metavar "SERVICE")
+  <*> orEmpty' (
+      (:) <$> argument (T.pack <$> str) (metavar "COMMAND")
+          <*> many (argument (T.pack <$> str) (metavar "ARG"))
+  )
+
+orEmpty' :: (Alternative f, Monoid a) => f a -> f a
+orEmpty' m = fromMaybe mempty <$> optional m
+
+runExec :: Bool -> Bool -> Maybe Text -> Bool -> Int -> [(Text, Text)] -> Maybe Text -> Text -> [Text] -> CommonOptions -> IO ()
+runExec detach privileged user noTTY index envs workDir service commandAndArgs opts = do
+  putErrText $ "Service: " <> service
+
+  ea <- defaultEvaluationArgs opts
+  Arion.Nix.withEvaluatedComposition ea $ \path -> do
+    commandAndArgs'' <- case commandAndArgs of
+      [] -> getDefaultExec path service
+      x -> pure x
+    let commandAndArgs' = case commandAndArgs'' of
+          [] -> ["/bin/sh"]
+          x -> x
+  
+    let args = concat
+          [ ["exec"]
+          , ("--detach" <$ guard detach :: [Text])
+          , "--privileged" <$ guard privileged
+          , "-T" <$ guard noTTY
+          , (\(k, v) -> ["--env", k <> "=" <> v]) =<< envs
+          , join $ toList (user <&> \u -> ["--user", u])
+          , ["--index", show index]
+          , join $ toList (workDir <&> \w -> ["--workdir", w])
+          , [service]
+          , commandAndArgs'
+          ]
+    DockerCompose.run DockerCompose.Args
+      { files = [path]
+      , otherArgs = args
+      }
+
+main :: IO ()
+main = 
+  (join . execParser) (info (parseAll <**> helper) fullDesc)
+  where
+    execParser = customExecParser (prefs showHelpOnEmpty)
+

src/haskell/lib/Arion/Aeson.hs

@@ -0,0 +1,29 @@
+module Arion.Aeson where
+
+import Prelude ()
+import           Data.Aeson
+import qualified Data.ByteString.Lazy          as BL
+import qualified Data.Text.Lazy                as TL
+import qualified Data.Text.Lazy.IO             as TL
+import qualified Data.Text.Lazy.Builder        as TB
+import qualified Data.Aeson.Encode.Pretty
+import           Data.Aeson.Encode.Pretty       ( defConfig
+                                                , keyOrder
+                                                , confCompare
+                                                , confTrailingNewline
+                                                )
+import           Protolude
+
+pretty :: ToJSON a => a -> Text
+pretty =
+  TL.toStrict
+    . TB.toLazyText
+    . Data.Aeson.Encode.Pretty.encodePrettyToTextBuilder' config
+  where config = defConfig { confCompare = compare, confTrailingNewline = True }
+
+decodeFile :: FromJSON a => FilePath -> IO a
+decodeFile fp = do
+  b <- BL.readFile fp
+  case eitherDecode b of
+    Left e -> panic (toS e)
+    Right v -> pure v

src/haskell/lib/Arion/DockerCompose.hs

@@ -0,0 +1,48 @@
+{-# LANGUAGE OverloadedStrings #-}
+module Arion.DockerCompose where
+
+import           Prelude                        ( )
+import           Protolude
+import           Arion.Aeson                    ( pretty )
+import           Data.Aeson
+import qualified Data.String
+import           System.Process
+import qualified Data.ByteString               as BS
+import qualified Data.ByteString.Lazy          as BL
+import           Paths_arion_compose
+import           Control.Applicative
+
+import qualified Data.Text                     as T
+import qualified Data.Text.IO                  as T
+
+import qualified Data.List.NonEmpty            as NE
+import           Data.List.NonEmpty             ( NonEmpty(..) )
+
+import           Control.Arrow                  ( (>>>) )
+import           System.IO.Temp                 ( withTempFile )
+import           System.IO                      ( hClose )
+
+data Args = Args
+  { files :: [FilePath]
+  , otherArgs :: [Text]
+  }
+
+run :: Args -> IO ()
+run args = do
+  let fileArgs = files args >>= \f -> ["--file", f]
+      allArgs  = fileArgs ++ map toS (otherArgs args)
+
+      procSpec = proc "docker-compose" allArgs
+
+  -- hPutStrLn stderr ("Running docker-compose with " <> show allArgs :: Text)
+
+  withCreateProcess procSpec $ \_in _out _err procHandle -> do
+
+    exitCode <- waitForProcess procHandle
+
+    case exitCode of
+      ExitSuccess -> pass
+      ExitFailure 1 -> exitFailure
+      e@ExitFailure {} -> do
+        throwIO $ FatalError $ "docker-compose failed with " <> show exitCode
+        exitWith e

src/haskell/lib/Arion/Images.hs

@@ -0,0 +1,61 @@
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE OverloadedStrings #-}
+module Arion.Images 
+  ( loadImages
+  ) where
+
+import Prelude()
+import Protolude hiding (to)
+
+import qualified Data.Aeson as Aeson
+import           Arion.Aeson (decodeFile)
+import qualified Data.ByteString as BS
+import qualified System.Process as Process
+
+import Control.Lens
+import Data.Aeson.Lens
+import Data.String
+import System.IO (withFile, IOMode(ReadMode))
+
+
+data Image = Image
+  { image :: Text -- ^ file path
+  , imageName :: Text
+  , imageTag :: Text
+  } deriving (Generic, Aeson.ToJSON, Aeson.FromJSON, Show)
+
+type TaggedImage = Text
+
+-- | Subject to change
+loadImages :: FilePath -> IO ()
+loadImages fp = do
+
+  v <- decodeFile fp
+
+  loaded <- dockerImages
+
+  let
+    images :: [Image]
+    images = (v :: Aeson.Value) ^.. key "x-arion" . key "images" . _Array . traverse . _JSON
+
+    isNew i = (imageName i <> ":" <> imageTag i) `notElem` loaded
+
+  traverse_ loadImage . map (toS . image) . filter isNew $ images
+
+loadImage :: FilePath -> IO ()
+loadImage imgPath = withFile (imgPath) ReadMode $ \fileHandle -> do
+  let procSpec = (Process.proc "docker" [ "load" ]) {
+          Process.std_in = Process.UseHandle fileHandle
+        }
+  Process.withCreateProcess procSpec $ \_in _out _err procHandle -> do
+    e <- Process.waitForProcess procHandle 
+    case e of
+      ExitSuccess -> pass
+      ExitFailure code -> panic $ "docker load (" <> show code <> ") failed for " <> toS imgPath
+
+
+dockerImages :: IO [TaggedImage]
+dockerImages = do
+  let procSpec = Process.proc "docker" [ "images",  "--filter", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}" ]
+  (map toS . lines) <$> Process.readCreateProcess procSpec ""

src/haskell/lib/Arion/Nix.hs

@@ -0,0 +1,190 @@
+{-# LANGUAGE OverloadedStrings #-}
+module Arion.Nix
+  ( evaluateComposition
+  , withEvaluatedComposition
+  , buildComposition
+  , withBuiltComposition
+  , replForComposition
+  , EvaluationArgs(..)
+  , EvaluationMode(..)
+  ) where
+
+import           Prelude                        ( )
+import           Protolude
+import           Arion.Aeson                    ( pretty )
+import           Data.Aeson
+import qualified Data.String
+import qualified System.Directory              as Directory
+import           System.Process
+import qualified Data.ByteString               as BS
+import qualified Data.ByteString.Lazy          as BL
+import           Paths_arion_compose
+import           Control.Applicative
+
+import qualified Data.Text                     as T
+import qualified Data.Text.IO                  as T
+
+import qualified Data.List.NonEmpty            as NE
+import           Data.List.NonEmpty             ( NonEmpty(..) )
+
+import           Control.Arrow                  ( (>>>) )
+import           System.IO.Temp                 ( withTempFile )
+import           System.IO                      ( hClose )
+
+data EvaluationMode =
+  ReadWrite | ReadOnly
+
+data EvaluationArgs = EvaluationArgs
+ { evalUid :: Int
+ , evalModules :: NonEmpty FilePath
+ , evalPkgs :: Text
+ , evalWorkDir :: Maybe FilePath
+ , evalMode :: EvaluationMode
+ , evalUserArgs :: [Text]
+ }
+
+evaluateComposition :: EvaluationArgs -> IO Value
+evaluateComposition ea = do
+  evalComposition <- getEvalCompositionFile
+  let commandArgs =
+        [ "--eval"
+        , "--strict"
+        , "--json"
+        , "--attr"
+        , "config.build.dockerComposeYamlAttrs"
+        ]
+      args =
+        [ evalComposition ]
+        ++ commandArgs
+        ++ modeArguments (evalMode ea)
+        ++ argArgs ea
+        ++ map toS (evalUserArgs ea)
+      procSpec = (proc "nix-instantiate" args)
+        { cwd = evalWorkDir ea
+        , std_out = CreatePipe
+        }
+  
+  withCreateProcess procSpec $ \_in outHM _err procHandle -> do
+    let outHandle = fromMaybe (panic "stdout missing") outHM
+
+    out <- BL.hGetContents outHandle
+
+    v <- Protolude.evaluate (eitherDecode out)
+
+    exitCode <- waitForProcess procHandle
+
+    case exitCode of
+      ExitSuccess -> pass
+      ExitFailure 1 -> exitFailure
+      e@ExitFailure {} -> do
+        throwIO $ FatalError $ "evaluation failed with " <> show exitCode
+        exitWith e
+
+    case v of
+      Right r -> pure r
+      Left  e -> throwIO $ FatalError "Couldn't parse nix-instantiate output"
+
+-- | Run with docker-compose.yaml tmpfile
+withEvaluatedComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r
+withEvaluatedComposition ea f = do
+  v <- evaluateComposition ea
+  withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path handle -> do
+    T.hPutStrLn handle (pretty v)
+    hClose handle
+    f path
+
+
+buildComposition :: FilePath -> EvaluationArgs -> IO ()
+buildComposition outLink ea = do
+  evalComposition <- getEvalCompositionFile
+  let commandArgs =
+        [ "--attr"
+        , "config.build.dockerComposeYaml"
+        , "--out-link"
+        , outLink
+        ]
+      args =
+        [ evalComposition ]
+        ++ commandArgs
+        ++ argArgs ea
+        ++ map toS (evalUserArgs ea)
+      procSpec = (proc "nix-build" args) { cwd = evalWorkDir ea }
+  
+  withCreateProcess procSpec $ \_in _out _err procHandle -> do
+
+    exitCode <- waitForProcess procHandle
+
+    case exitCode of
+      ExitSuccess -> pass
+      ExitFailure 1 -> exitFailure
+      e@ExitFailure {} -> do
+        throwIO $ FatalError $ "nix-build failed with " <> show exitCode
+        exitWith e
+
+-- | Do something with a docker-compose.yaml.
+withBuiltComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r
+withBuiltComposition ea f = do
+  withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path handle -> do
+    hClose handle
+    -- Known problem: kills atomicity of withTempFile; won't fix because we should manage gc roots,
+    -- impl of which will probably avoid this "problem". It seems unlikely to cause issues.
+    Directory.removeFile path
+    buildComposition path ea
+    f path
+
+
+replForComposition :: EvaluationArgs -> IO ()
+replForComposition ea = do
+    evalComposition <- getEvalCompositionFile
+    let args =
+          [ "repl", evalComposition ]
+          ++ argArgs ea
+          ++ map toS (evalUserArgs ea)
+        procSpec = (proc "nix" args) { cwd = evalWorkDir ea }
+    
+    withCreateProcess procSpec $ \_in _out _err procHandle -> do
+
+      exitCode <- waitForProcess procHandle
+
+      case exitCode of
+        ExitSuccess -> pass
+        ExitFailure 1 -> exitFailure
+        e@ExitFailure {} -> do
+          throwIO $ FatalError $ "nix repl failed with " <> show exitCode
+          exitWith e
+
+argArgs :: EvaluationArgs -> [[Char]]
+argArgs ea =
+      [ "--argstr"
+      , "uid"
+      , show $ evalUid ea
+      , "--arg"
+      , "modules"
+      , modulesNixExpr $ evalModules ea
+      , "--arg"
+      , "pkgs"
+      , toS $ evalPkgs ea
+      ]
+
+getEvalCompositionFile :: IO FilePath
+getEvalCompositionFile = getDataFileName "nix/eval-composition.nix"
+
+modeArguments :: EvaluationMode -> [[Char]]
+modeArguments ReadWrite = [ "--read-write-mode" ]
+modeArguments ReadOnly = [ "--readonly-mode" ]
+
+modulesNixExpr :: NonEmpty FilePath -> [Char]
+modulesNixExpr =
+  NE.toList >>> fmap pathExpr >>> Data.String.unwords >>> wrapList
+ where
+  pathExpr :: FilePath -> [Char]
+  pathExpr path | isAbsolute path = "(/. + \"/${" <> toNixStringLiteral path <> "}\")"
+                | otherwise       = "(./. + \"/${" <> toNixStringLiteral path <> "}\")"
+
+  isAbsolute ('/' : _) = True
+  isAbsolute _         = False
+
+  wrapList s = "[ " <> s <> " ]"
+
+toNixStringLiteral :: [Char] -> [Char]
+toNixStringLiteral = show -- FIXME: custom escaping including '$'

src/haskell/lib/Arion/Services.hs

@@ -0,0 +1,27 @@
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE OverloadedStrings #-}
+module Arion.Services
+  ( getDefaultExec
+  ) where
+
+import Prelude()
+import Protolude hiding (to)
+
+import qualified Data.Aeson as Aeson
+import           Arion.Aeson (decodeFile)
+import qualified Data.ByteString as BS
+import qualified System.Process as Process
+
+import Control.Lens
+import Data.Aeson.Lens
+import Data.String
+import System.IO (withFile, IOMode(ReadMode))
+
+-- | Subject to change
+getDefaultExec :: FilePath -> Text -> IO [Text]
+getDefaultExec fp service = do
+
+  v <- decodeFile fp
+
+  pure ((v :: Aeson.Value) ^.. key "x-arion" . key "serviceInfo" . key service . key "defaultExec" . _Array . traverse . _String)

src/haskell/test/Arion/NixSpec.hs

@@ -0,0 +1,64 @@
+{-# LANGUAGE OverloadedStrings #-}
+module Arion.NixSpec
+  ( spec
+  )
+where
+
+import           Protolude
+import           Test.Hspec
+import           Test.QuickCheck
+import qualified Data.List.NonEmpty            as NEL
+import           Arion.Aeson
+import           Arion.Nix
+import qualified Data.Text                     as T
+import qualified Data.Text.IO                  as T
+import qualified Data.Text.Lazy.IO             as TL
+import qualified Data.Text.Lazy.Builder        as TB
+import qualified Data.Aeson.Encode.Pretty
+import Data.Char (isSpace)
+
+spec :: Spec
+spec = describe "evaluateComposition" $ it "matches an example" $ do
+  x <- Arion.Nix.evaluateComposition EvaluationArgs
+    { evalUid      = 123
+    , evalModules  = NEL.fromList
+                       ["src/haskell/testdata/Arion/NixSpec/arion-compose.nix"]
+    , evalPkgs     = "import <nixpkgs> {}"
+    , evalWorkDir  = Nothing
+    , evalMode     = ReadOnly
+    , evalUserArgs = ["--show-trace"]
+    }
+  let actual = pretty x
+  expected <- T.readFile "src/haskell/testdata/Arion/NixSpec/arion-compose.json"
+  censorPaths actual `shouldBe` censorPaths expected
+
+censorPaths = censorImages . censorStorePaths
+--censorPaths = censorStorePaths
+
+censorStorePaths :: Text -> Text
+censorStorePaths x = case T.breakOn "/nix/store/" x of
+  (prefix, tl) | (tl :: Text) == "" -> prefix
+  (prefix, tl) -> prefix <> "<STOREPATH>" <> censorPaths
+    (T.dropWhile isNixNameChar $ T.drop (T.length "/nix/store/") tl)
+
+-- Probably slow, due to not O(1) <>
+censorImages :: Text -> Text
+censorImages x = case T.break (\c -> c == ':' || c == '"') x of
+  (prefix, tl) | tl == "" -> prefix
+  (prefix, tl) | let imageId = T.take 33 (T.drop 1 tl)
+               , T.last imageId == '\"'
+                 -- Approximation of nix hash validation
+               , T.all (\c -> (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) (T.take 32 imageId)
+               -> prefix <> T.take 1 tl <> "<HASH>" <> censorImages (T.drop 33 tl)
+  (prefix, tl) -> prefix <> T.take 1 tl <> censorImages (T.drop 1 tl)
+
+
+-- | WARNING: THIS IS LIKELY WRONG: DON'T REUSE
+isNixNameChar :: Char -> Bool
+isNixNameChar c | c >= '0' && c <= '9' = True
+isNixNameChar c | c >= 'a' && c <= 'z' = True
+isNixNameChar c | c >= 'A' && c <= 'Z' = True
+isNixNameChar c | c == '-' = True
+isNixNameChar c | c == '.' = True
+isNixNameChar c | c == '_' = True -- WRONG?
+isNixNameChar c = False -- WRONG?

src/haskell/test/Spec.hs

@@ -0,0 +1,11 @@
+module Spec
+  ( spec
+  )
+where
+
+import           Test.Hspec
+import qualified Arion.NixSpec
+
+spec :: Spec
+spec = do
+  describe "Arion.Nix" Arion.NixSpec.spec

src/haskell/test/TestMain.hs

@@ -0,0 +1,9 @@
+module Main where
+
+import           Protolude
+import           Test.Hspec.Runner
+import qualified Spec
+
+main :: IO ()
+main = hspecWith config Spec.spec
+  where config = defaultConfig { configColorMode = ColorAlways }

src/haskell/testdata/Arion/NixSpec/arion-compose.json

@@ -0,0 +1,49 @@
+{
+    "services": {
+        "webserver": {
+            "command": [
+                "/nix/store/b9w61w4g8sqgrm3rid6ca22krslqghb3-nixos-system-unnamed-19.03.173100.e726e8291b2/init"
+            ],
+            "environment": {
+                "NIX_REMOTE": "",
+                "PATH": "/usr/bin:/run/current-system/sw/bin/",
+                "container": "docker"
+            },
+            "image": "arion-base:<HASH>",
+            "ports": [
+                "8000:80"
+            ],
+            "stop_signal": "SIGRTMIN+3",
+            "sysctls": {},
+            "tmpfs": [
+                "/run",
+                "/run/wrappers",
+                "/tmp:exec,mode=777"
+            ],
+            "tty": true,
+            "volumes": [
+                "/sys/fs/cgroup:/sys/fs/cgroup:ro",
+                "/nix/store:/nix/store:ro",
+                "/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system:ro"
+            ]
+        }
+    },
+    "version": "3.4",
+    "x-arion": {
+        "images": [
+            {
+                "image": "<STOREPATH>",
+                "imageName": "arion-base",
+                "imageTag": "<HASH>"
+            }
+        ],
+        "serviceInfo": {
+            "webserver": {
+                "defaultExec": [
+                    "/run/current-system/sw/bin/bash",
+                    "-l"
+                ]
+            }
+        }
+    }
+}

src/haskell/testdata/Arion/NixSpec/arion-compose.nix

@@ -0,0 +1,12 @@
+{
+  docker-compose.services.webserver = { pkgs, ... }: {
+    nixos.useSystemd = true;
+    nixos.configuration.boot.tmpOnTmpfs = true;
+    nixos.configuration.services.nginx.enable = true;
+    nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual";
+    service.useHostStore = true;
+    service.ports = [
+      "8000:80" # host:container
+    ];
+  };
+}

src/haskell/testdata/docker-compose-example.json

@@ -0,0 +1,42 @@
+{
+  "services": {
+    "webserver": {
+      "environment": {
+        "container": "docker"
+      },
+      "image": "webserver:xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70",
+      "ports": [
+        "8000:80"
+      ],
+      "stop_signal": "SIGRTMIN+3",
+      "sysctls": {},
+      "tmpfs": [
+        "/run",
+        "/run/wrappers",
+        "/tmp:exec,mode=777"
+      ],
+      "tty": true,
+      "volumes": [
+        "/sys/fs/cgroup:/sys/fs/cgroup:ro"
+      ]
+    }
+  },
+  "version": "3.4",
+  "x-arion": {
+    "images": [
+      {
+        "image": "/nix/store/xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70-docker-image-webserver.tar.gz",
+        "imageName": "webserver",
+        "imageTag": "xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70"
+      }
+    ],
+    "serviceInfo": {
+      "webserver": {
+        "defaultExec": [
+          "/run/current-system/sw/bin/bash",
+          "-l"
+        ]
+      }
+    }
+  }
+}

src/nix/eval-composition.nix

@@ -22,6 +22,7 @@ let
     ./modules/composition/host-environment.nix
     ./modules/composition/images.nix
     ./modules/composition/service-info.nix
+    ./modules/composition/arion-base-image.nix
   ];
 
   argsModule = {

src/nix/eval-service.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, ... }:
 
-{ modules, host, name }:
+{ modules, host, name, composition }:
 let
   composite = lib.evalModules {
     check = true;
@@ -13,7 +13,7 @@ let
     ./modules/service/docker-compose-service.nix
     ./modules/service/extended-info.nix
     ./modules/service/host-store.nix
-    ./modules/service/host.nix
+    ./modules/service/context.nix
     ./modules/service/image.nix
     ./modules/service/nixos.nix
     ./modules/service/nixos-init.nix
@@ -25,6 +25,7 @@ let
     config._module.args.pkgs = lib.mkForce pkgs;
     config.host = host;
     config.service.name = name;
+    config.composition = composition;
   };
 
 in

src/nix/modules/composition/arion-base-image.nix

@@ -0,0 +1,41 @@
+
+
+# This module is subject to change.
+# In particular, arion-base should use a generic non-service image building system
+
+{ config, lib, pkgs, ... }: 
+
+let
+
+  tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath));
+  name = "arion-base";
+
+  builtImage = pkgs.dockerTools.buildLayeredImage {
+    inherit name;
+    contents = pkgs.runCommand "minimal-contents" {} ''
+        mkdir -p $out/bin $out/usr/bin
+        ln -s /run/system/bin/sh $out/bin/sh
+        ln -s /run/system/usr/bin/env $out/usr/bin/env
+      '';
+    config = {};
+  };
+
+in
+
+{
+
+  options = {
+    arionBaseImage = lib.mkOption {
+      type = lib.types.str;
+      description = "Image to use when using useHostStore. Don't use this option yourself. It's going away.";
+      internal = true;
+    };
+  };
+
+  config = {
+    arionBaseImage = "${name}:${tag}";
+    build.imagesToLoad = lib.mkIf (lib.any (s: s.config.service.useHostStore) (lib.attrValues config.docker-compose.evaluatedServices)) [
+      { image = builtImage; imageName = name; imageTag = tag; }
+    ];  
+  };
+}

src/nix/modules/composition/docker-compose.nix

@@ -11,7 +11,11 @@
  */
 { pkgs, lib, config, ... }:
 let
-  evalService = name: modules: pkgs.callPackage ../../eval-service.nix {} { inherit name modules; inherit (config) host; };
+  evalService = name: modules: pkgs.callPackage ../../eval-service.nix {} {
+    inherit name modules;
+    inherit (config) host;
+    composition = config;
+  };
 
 in
 {
@@ -19,10 +23,17 @@ in
     build.dockerComposeYaml = lib.mkOption {
       type = lib.types.package;
       description = "A derivation that produces a docker-compose.yaml file for this composition.";
+      readOnly = true;
     };
     build.dockerComposeYamlText = lib.mkOption {
       type = lib.types.str;
       description = "The text of build.dockerComposeYaml.";
+      readOnly = true;
+    };
+    build.dockerComposeYamlAttrs = lib.mkOption {
+      type = lib.types.attrsOf lib.types.unspecified;
+      description = "The text of build.dockerComposeYaml.";
+      readOnly = true;
     };
     docker-compose.raw = lib.mkOption {
       type = lib.types.attrs;
@@ -45,7 +56,8 @@ in
   };
   config = {
     build.dockerComposeYaml = pkgs.writeText "docker-compose.yaml" config.build.dockerComposeYamlText;
-    build.dockerComposeYamlText = builtins.toJSON (config.docker-compose.raw);
+    build.dockerComposeYamlText = builtins.toJSON (config.build.dockerComposeYamlAttrs);
+    build.dockerComposeYamlAttrs = config.docker-compose.raw;
 
     docker-compose.evaluatedServices = lib.mapAttrs evalService config.docker-compose.services;
     docker-compose.raw = {

src/nix/modules/service/context.nix

@@ -7,5 +7,11 @@
         The composition-level host option values.
       '';
     };
+    composition = lib.mkOption {
+      type = lib.types.attrs;
+      description = ''
+        The composition configuration.
+      '';
+    };
   };
 }

src/nix/modules/service/host-store.nix

@@ -30,8 +30,7 @@ in
   };
   config = mkIf config.service.useHostStore {
     image.nixBuild = false; # no need to build and load
-    service.image = "arion-base";
+    service.image = config.composition.arionBaseImage;
-    service.build.context = "${../../../arion-image}";
     service.environment.NIX_REMOTE = lib.optionalString config.service.useHostNixDaemon "daemon";
     service.volumes = [
       "${config.host.nixStorePrefix}/nix/store:/nix/store${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"

tests/arion-test/default.nix

@@ -12,7 +12,6 @@ in
   machine = { pkgs, lib, ... }: {
     environment.systemPackages = [
       pkgs.arion
-      pkgs.docker-compose
     ];
     virtualisation.docker.enable = true;