From 9443fe8410335d28c31c85a7cd6f92f704d1f0e4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 14 Jun 2019 16:10:37 +0200 Subject: [PATCH 01/27] Add Haskell package --- .gitignore | 3 ++ CHANGELOG.md | 5 +++ Setup.hs | 2 ++ arion-compose.cabal | 54 +++++++++++++++++++++++++++++++ live-unit-tests | 13 ++++++++ nix/haskell-overlay.nix | 3 ++ nix/overlay.nix | 17 +++++++++- shell.nix | 1 + src/haskell/exe/Main.hs | 4 +++ src/haskell/test/Arion/FooSpec.hs | 11 +++++++ src/haskell/test/Spec.hs | 11 +++++++ src/haskell/test/TestMain.hs | 9 ++++++ 12 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md create mode 100644 Setup.hs create mode 100644 arion-compose.cabal create mode 100755 live-unit-tests create mode 100644 nix/haskell-overlay.nix create mode 100644 shell.nix create mode 100644 src/haskell/exe/Main.hs create mode 100644 src/haskell/test/Arion/FooSpec.hs create mode 100644 src/haskell/test/Spec.hs create mode 100644 src/haskell/test/TestMain.hs diff --git a/.gitignore b/.gitignore index 750baeb..c372372 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ result result-* + +dist/ +dist-newstyle/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..405dedc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Revision history for arion-compose + +## 0.1.0.0 -- YYYY-mm-dd + +* First version. Released on an unsuspecting world. diff --git a/Setup.hs b/Setup.hs new file mode 100644 index 0000000..9a994af --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/arion-compose.cabal b/arion-compose.cabal new file mode 100644 index 0000000..825b04c --- /dev/null +++ b/arion-compose.cabal @@ -0,0 +1,54 @@ +cabal-version: 2.2 + +name: arion-compose +version: 0.1.0.0 +synopsis: Run docker-compose with help from Nix/NixOS +-- description: +homepage: https://github.com/hercules-ci/arion#readme +-- bug-reports: +license: Apache-2.0 +license-file: LICENSE +author: Robert Hensing +maintainer: robert@hercules-ci.com +-- copyright: +-- category: +extra-source-files: CHANGELOG.md, README.asciidoc +write-ghc-enviroment-files: + never + +common deps + build-depends: base ^>=4.12.0.0 + , aeson + , protolude + + +library + import: deps + -- exposed-modules: + -- other-modules: + -- other-extensions: + build-depends: process + hs-source-dirs: src/haskell/lib + default-language: Haskell2010 + +executable arion + import: deps + main-is: Main.hs + -- other-modules: + -- other-extensions: + build-depends: optparse-applicative + , process + hs-source-dirs: src/haskell/exe + default-language: Haskell2010 + +test-suite arion-unit-tests + import: deps + type: exitcode-stdio-1.0 + main-is: Main.hs + -- other-modules: + -- other-extensions: + build-depends: arion-compose + , hspec + , QuickCheck + hs-source-dirs: src/haskell/test + default-language: Haskell2010 diff --git a/live-unit-tests b/live-unit-tests new file mode 100755 index 0000000..4a2cc6c --- /dev/null +++ b/live-unit-tests @@ -0,0 +1,13 @@ +#!/usr/bin/env nix-shell +#!nix-shell ./shell.nix +#!nix-shell -i bash +set -eux -o pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +ghcid \ + --command 'ghci -isrc/haskell/exe -isrc/haskell/lib -isrc/haskell/test src/haskell/test/TestMain.hs' \ + --test=Main.main \ + --restart=hercules-ci-api.cabal \ + --restart=../stack.yaml \ + ; diff --git a/nix/haskell-overlay.nix b/nix/haskell-overlay.nix new file mode 100644 index 0000000..9de9363 --- /dev/null +++ b/nix/haskell-overlay.nix @@ -0,0 +1,3 @@ +self: super: hself: hsuper: { + arion-compose = hself.callCabal2nix "arion-compose" ./.. {}; +} \ No newline at end of file diff --git a/nix/overlay.nix b/nix/overlay.nix index 13ad5dc..30a6b7a 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,5 +1,20 @@ -self: super: { +self: super: +let + inherit (self.arion-project) haskellPkgs; +in +{ arion = super.callPackage ../arion.nix {}; tests = super.callPackage ../tests {}; doc = super.callPackage ../doc {}; + + arion-project = super.recurseIntoAttrs { + haskellPkgs = super.haskellPackages.extend (import ./haskell-overlay.nix self super); + shell = haskellPkgs.shellFor { + packages = p: [p.arion-compose]; + buildInputs = [ + haskellPkgs.cabal-install + haskellPkgs.ghcid + ]; + }; + }; } diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..14e3ffe --- /dev/null +++ b/shell.nix @@ -0,0 +1 @@ +(import ./nix {}).arion-project.shell diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs new file mode 100644 index 0000000..65ae4a0 --- /dev/null +++ b/src/haskell/exe/Main.hs @@ -0,0 +1,4 @@ +module Main where + +main :: IO () +main = putStrLn "Hello, Haskell!" diff --git a/src/haskell/test/Arion/FooSpec.hs b/src/haskell/test/Arion/FooSpec.hs new file mode 100644 index 0000000..d4ea5fd --- /dev/null +++ b/src/haskell/test/Arion/FooSpec.hs @@ -0,0 +1,11 @@ +module Arion.FooSpec + ( spec + ) +where + +import Test.Hspec +import Test.QuickCheck + +spec :: Spec +spec = do + it "foo" $ property True diff --git a/src/haskell/test/Spec.hs b/src/haskell/test/Spec.hs new file mode 100644 index 0000000..73ab5ab --- /dev/null +++ b/src/haskell/test/Spec.hs @@ -0,0 +1,11 @@ +module Spec + ( spec + ) +where + +import Test.Hspec +import qualified Arion.FooSpec + +spec :: Spec +spec = do + describe "Arion.Foo" Arion.FooSpec.spec diff --git a/src/haskell/test/TestMain.hs b/src/haskell/test/TestMain.hs new file mode 100644 index 0000000..746b8d7 --- /dev/null +++ b/src/haskell/test/TestMain.hs @@ -0,0 +1,9 @@ +module Main where + +import Protolude +import Test.Hspec.Runner +import qualified Spec + +main :: IO () +main = hspecWith config Spec.spec + where config = defaultConfig { configColorMode = ColorAlways } From ba6fa62c4a278abc21d0b577686f2af7fe954f1e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 14 Jun 2019 18:10:37 +0200 Subject: [PATCH 02/27] Add docker compose example for parsing unit test --- .../testdata/docker-compose-example.json | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/haskell/testdata/docker-compose-example.json diff --git a/src/haskell/testdata/docker-compose-example.json b/src/haskell/testdata/docker-compose-example.json new file mode 100644 index 0000000..abcdd61 --- /dev/null +++ b/src/haskell/testdata/docker-compose-example.json @@ -0,0 +1,42 @@ +{ + "services": { + "webserver": { + "environment": { + "container": "docker" + }, + "image": "webserver:xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70", + "ports": [ + "8000:80" + ], + "stop_signal": "SIGRTMIN+3", + "sysctls": {}, + "tmpfs": [ + "/run", + "/run/wrappers", + "/tmp:exec,mode=777" + ], + "tty": true, + "volumes": [ + "/sys/fs/cgroup:/sys/fs/cgroup:ro" + ] + } + }, + "version": "3.4", + "x-arion": { + "images": [ + { + "image": "/nix/store/xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70-docker-image-webserver.tar.gz", + "imageName": "webserver", + "imageTag": "xr4ljmz3qfcwlq9rl4mr4qdrzw93rl70" + } + ], + "serviceInfo": { + "webserver": { + "defaultExec": [ + "/run/current-system/sw/bin/bash", + "-l" + ] + } + } + } +} From 61cc348281f54bbd70432a67c3075e26eb7b1d9e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 15 Jun 2019 14:32:59 +0200 Subject: [PATCH 03/27] Pass system through project nix files --- default.nix | 2 +- nix/default.nix | 12 +++++++++--- shell.nix | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/default.nix b/default.nix index 2570b9f..7f48895 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,4 @@ -args@{ pkgs ? import ./nix args, ... }: +args@{ pkgs ? import ./nix args, system ? null, ... }: { inherit (pkgs) arion tests; diff --git a/nix/default.nix b/nix/default.nix index 4c25cf1..c3fd071 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,8 +1,12 @@ /** * This is the entry-point for all nix execution in this project. */ -{ nixpkgsSrc ? ./nixpkgs.nix, ... }: -import (import ./nixpkgs.nix) { +{ nixpkgsSrc ? ./nixpkgs.nix +, system ? null +, ... +}: + +import (import ./nixpkgs.nix) ({ # Makes the config pure as well. See /top-level/impure.nix: config = { }; @@ -10,4 +14,6 @@ import (import ./nixpkgs.nix) { # all the packages are defined there: (import ./overlay.nix) ]; -} +} // (if system == null then {} else { + inherit system; +})) diff --git a/shell.nix b/shell.nix index 14e3ffe..f6c0ea0 100644 --- a/shell.nix +++ b/shell.nix @@ -1 +1 @@ -(import ./nix {}).arion-project.shell +args@{...}: (import ./nix args).arion-project.shell From 01f73e486b33e948f8da10bfe134ae9394121095 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 15 Jun 2019 15:45:06 +0200 Subject: [PATCH 04/27] Build the examples explicitly on/for Linux --- examples/full-nixos/arion-pkgs.nix | 6 +++++- examples/minimal/arion-pkgs.nix | 6 +++++- examples/nixos-unit/arion-pkgs.nix | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/examples/full-nixos/arion-pkgs.nix b/examples/full-nixos/arion-pkgs.nix index d13a605..69aad13 100644 --- a/examples/full-nixos/arion-pkgs.nix +++ b/examples/full-nixos/arion-pkgs.nix @@ -1,2 +1,6 @@ # Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH -import {} +import { + # We specify the architecture explicitly. Use a Linux remote builder when + # calling arion from other platforms. + system = "x86_64-linux"; +} diff --git a/examples/minimal/arion-pkgs.nix b/examples/minimal/arion-pkgs.nix index d13a605..69aad13 100644 --- a/examples/minimal/arion-pkgs.nix +++ b/examples/minimal/arion-pkgs.nix @@ -1,2 +1,6 @@ # Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH -import {} +import { + # We specify the architecture explicitly. Use a Linux remote builder when + # calling arion from other platforms. + system = "x86_64-linux"; +} diff --git a/examples/nixos-unit/arion-pkgs.nix b/examples/nixos-unit/arion-pkgs.nix index d13a605..69aad13 100644 --- a/examples/nixos-unit/arion-pkgs.nix +++ b/examples/nixos-unit/arion-pkgs.nix @@ -1,2 +1,6 @@ # Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH -import {} +import { + # We specify the architecture explicitly. Use a Linux remote builder when + # calling arion from other platforms. + system = "x86_64-linux"; +} From 9b047987aefd860dc0cf036a8e9448e12a7ceebf Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Sat, 15 Jun 2019 12:35:48 +0200 Subject: [PATCH 05/27] Add basic command line parsing --- arion-compose.cabal | 1 + live-check | 13 +++ live-unit-tests | 1 + src/haskell/exe/Main.hs | 196 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 209 insertions(+), 2 deletions(-) create mode 100755 live-check diff --git a/arion-compose.cabal b/arion-compose.cabal index 825b04c..eb631c0 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -19,6 +19,7 @@ write-ghc-enviroment-files: common deps build-depends: base ^>=4.12.0.0 , aeson + , text , protolude diff --git a/live-check b/live-check new file mode 100755 index 0000000..d7cbfb2 --- /dev/null +++ b/live-check @@ -0,0 +1,13 @@ +#!/usr/bin/env nix-shell +#!nix-shell ./shell.nix +#!nix-shell -i bash +set -eux -o pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" + +ghcid \ + --command 'ghci -isrc/haskell/exe src/haskell/exe/Main.hs' \ + --reload=src/haskell \ + --restart=hercules-ci-api.cabal \ + --restart=../stack.yaml \ + ; diff --git a/live-unit-tests b/live-unit-tests index 4a2cc6c..09e3af9 100755 --- a/live-unit-tests +++ b/live-unit-tests @@ -8,6 +8,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" ghcid \ --command 'ghci -isrc/haskell/exe -isrc/haskell/lib -isrc/haskell/test src/haskell/test/TestMain.hs' \ --test=Main.main \ + --reload=src/haskell \ --restart=hercules-ci-api.cabal \ --restart=../stack.yaml \ ; diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index 65ae4a0..353195b 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -1,4 +1,196 @@ -module Main where +{-# LANGUAGE NoImplicitPrelude #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ApplicativeDo #-} +{-# LANGUAGE OverloadedStrings #-} + +import Protolude hiding (Down) + +import Options.Applicative +import Control.Applicative + +import qualified Data.Text as T +import qualified Data.Text.IO as T + +import qualified Data.List.NonEmpty as NE +import Data.List.NonEmpty (NonEmpty(..)) + +import Control.Arrow ((>>>)) + + +data CommonOptions = + CommonOptions + { files :: NonEmpty FilePath + , pkgs :: Text + } + deriving (Show) + + +newtype DockerComposeArgs = + DockerComposeArgs { unDockerComposeArgs :: [Text] } + + +data DockerComposeCmd = + Build + | Bundle + | Config + | Create + | Down + | Events + | Exec + | Help + | Images + | Kill + | Logs + | Pause + | Port + | Ps + | Pull + | Push + | Restart + | Rm + | Run + | Scale + | Start + | Stop + | Top + | Unpause + | Up + | Version + deriving (Show) + + + +ensureConfigFile :: [FilePath] -> NonEmpty FilePath +ensureConfigFile [] = "./arion-compose.nix" :| [] +ensureConfigFile (x:xs) = x :| xs + + +parseOptions :: Parser CommonOptions +parseOptions = do + files <- + ensureConfigFile <$> + many (strOption + ( short 'f' + <> long "file" + <> metavar "FILE" + <> help "Use FILE instead of the default ./arion-compose.nix. \ + \Can be specified multiple times for a merged configuration" )) + pkgs <- T.pack <$> strOption + ( short 'p' + <> long "pkgs" + <> metavar "EXPR" + <> showDefault + <> value "./arion-pkgs.nix" + <> help "Use EXPR to get the Nixpkgs attrset used for bootstrapping \ + \and evaluating the configuration." ) + pure CommonOptions{..} + + +parseCommand :: Parser (CommonOptions -> IO ()) +parseCommand = + hsubparser + ( command "cat" (info (pure runCat) fullDesc) + <> command "repl" (info (pure runRepl) fullDesc) + <> command "exec" (info (pure runExec) fullDesc) + ) + <|> + hsubparser + ( dcParser "build" Build "Build or rebuild services" + <> dcParser "bundle" Bundle "Generate a Docker bundle from the Compose file" + <> dcParser "config" Config "Validate and view the Compose file" + <> dcParser "create" Create "Create services" + <> dcParser "down" Down "Stop and remove containers, networks, images, and volumes" + <> dcParser "events" Events "Receive real time events from containers" + <> dcParser "exec" Exec "Execute a command in a running container" + <> dcParser "help" Help "Get help on a command" + <> dcParser "images" Images "List images" + <> dcParser "kill" Kill "Kill containers" + <> dcParser "logs" Logs "View output from containers" + <> dcParser "pause" Pause "Pause services" + <> dcParser "port" Port "Print the public port for a port binding" + <> dcParser "ps" Ps "List containers" + <> dcParser "pull" Pull "Pull service images" + <> dcParser "push" Push "Push service images" + <> dcParser "restart" Restart "Restart services" + <> dcParser "rm" Rm "Remove stopped containers" + <> dcParser "run" Run "Run a one-off command" + <> dcParser "scale" Scale "Set number of containers for a service" + <> dcParser "start" Start "Start services" + <> dcParser "stop" Stop "services" + <> dcParser "top" Top "Display the running processes" + <> dcParser "unpause" Unpause "Unpause services" + <> dcParser "up" Up "Create and start containers" + <> dcParser "version" Version "Show the Docker-Compose version information" + + <> metavar "DOCKER-COMPOSE-COMMAND" + <> commandGroup "Docker Compose Commands:" + ) + + +dcParser + :: Text + -> DockerComposeCmd + -> Text + -> Mod CommandFields (CommonOptions -> IO ()) +dcParser cmdStr cmd help = + command + (T.unpack cmdStr) + (info + (runDockerCompose <$> pure cmd <*> parseDockerComposeArgs) + (progDesc (T.unpack help) <> fullDesc <> forwardOptions)) + + +parseAll :: Parser (IO ()) +parseAll = + flip ($) <$> parseOptions <*> parseCommand + + +parseDockerComposeArgs :: Parser DockerComposeArgs +parseDockerComposeArgs = + DockerComposeArgs <$> + many (argument (T.pack <$> str) (metavar "DOCKER-COMPOSE ARGS...")) + + +shouldEval :: DockerComposeCmd -> Bool +shouldEval Up = True +shouldEval Down = True + + +runDockerCompose :: DockerComposeCmd -> DockerComposeArgs -> CommonOptions -> IO () +runDockerCompose cmd args opts = T.putStrLn (show cmd) + + +runCat :: CommonOptions -> IO () +runCat (CommonOptions files pkgs) = do + T.putStrLn "Running cat ... TODO" + T.putStrLn (modulesNixExpr files) + + +runRepl :: CommonOptions -> IO () +runRepl opts = T.putStrLn "Running repl ... TODO" + + +runExec :: CommonOptions -> IO () +runExec opts = T.putStrLn "Running exec ... TODO" + + +modulesNixExpr :: NonEmpty FilePath -> Text +modulesNixExpr = + NE.toList + >>> fmap pathExpr + >>> T.unwords + >>> wrapList + where + pathExpr path | isAbsolute path = "(/. + \"" <> T.pack path <> "\")" + | otherwise = "(./. + \"" <> T.pack path <> "\")" + + isAbsolute ('/':_) = True + isAbsolute _ = False + + wrapList s = "[ " <> s <> " ]" + main :: IO () -main = putStrLn "Hello, Haskell!" +main = + (join . execParser) (info (parseAll <**> helper) fullDesc) + From 60cb5cb5c39b4ed17fc9040fda4c67abfbebee0f Mon Sep 17 00:00:00 2001 From: Stefan Matting Date: Sat, 15 Jun 2019 21:13:09 +0200 Subject: [PATCH 06/27] refactor command line parsing --- src/haskell/exe/Main.hs | 173 +++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 102 deletions(-) diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index 353195b..fe4f608 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -16,7 +16,6 @@ import Data.List.NonEmpty (NonEmpty(..)) import Control.Arrow ((>>>)) - data CommonOptions = CommonOptions { files :: NonEmpty FilePath @@ -24,47 +23,13 @@ data CommonOptions = } deriving (Show) - newtype DockerComposeArgs = DockerComposeArgs { unDockerComposeArgs :: [Text] } - -data DockerComposeCmd = - Build - | Bundle - | Config - | Create - | Down - | Events - | Exec - | Help - | Images - | Kill - | Logs - | Pause - | Port - | Ps - | Pull - | Push - | Restart - | Rm - | Run - | Scale - | Start - | Stop - | Top - | Unpause - | Up - | Version - deriving (Show) - - - ensureConfigFile :: [FilePath] -> NonEmpty FilePath ensureConfigFile [] = "./arion-compose.nix" :| [] ensureConfigFile (x:xs) = x :| xs - parseOptions :: Parser CommonOptions parseOptions = do files <- @@ -85,94 +50,68 @@ parseOptions = do \and evaluating the configuration." ) pure CommonOptions{..} - parseCommand :: Parser (CommonOptions -> IO ()) parseCommand = hsubparser - ( command "cat" (info (pure runCat) fullDesc) - <> command "repl" (info (pure runRepl) fullDesc) - <> command "exec" (info (pure runExec) fullDesc) + ( command "cat" (info (pure runCat) (progDesc "TODO: cat doc" <> fullDesc)) + <> command "repl" (info (pure runRepl) (progDesc "TODO: repl doc" <> fullDesc)) + <> command "exec" (info (pure runExec) (progDesc "TODO: exec doc" <> fullDesc)) ) <|> hsubparser - ( dcParser "build" Build "Build or rebuild services" - <> dcParser "bundle" Bundle "Generate a Docker bundle from the Compose file" - <> dcParser "config" Config "Validate and view the Compose file" - <> dcParser "create" Create "Create services" - <> dcParser "down" Down "Stop and remove containers, networks, images, and volumes" - <> dcParser "events" Events "Receive real time events from containers" - <> dcParser "exec" Exec "Execute a command in a running container" - <> dcParser "help" Help "Get help on a command" - <> dcParser "images" Images "List images" - <> dcParser "kill" Kill "Kill containers" - <> dcParser "logs" Logs "View output from containers" - <> dcParser "pause" Pause "Pause services" - <> dcParser "port" Port "Print the public port for a port binding" - <> dcParser "ps" Ps "List containers" - <> dcParser "pull" Pull "Pull service images" - <> dcParser "push" Push "Push service images" - <> dcParser "restart" Restart "Restart services" - <> dcParser "rm" Rm "Remove stopped containers" - <> dcParser "run" Run "Run a one-off command" - <> dcParser "scale" Scale "Set number of containers for a service" - <> dcParser "start" Start "Start services" - <> dcParser "stop" Stop "services" - <> dcParser "top" Top "Display the running processes" - <> dcParser "unpause" Unpause "Unpause services" - <> dcParser "up" Up "Create and start containers" - <> dcParser "version" Version "Show the Docker-Compose version information" + ( commandDC runBuildAndDC "build" "Build or rebuild services" + <> commandDC runBuildAndDC "bundle" "Generate a Docker bundle from the Compose file" + <> commandDC runEvalAndDC "config" "Validate and view the Compose file" + <> commandDC runBuildAndDC "create" "Create services" + <> commandDC runEvalAndDC "down" "Stop and remove containers, networks, images, and volumes" + <> commandDC runEvalAndDC "events" "Receive real time events from containers" + <> commandDC runEvalAndDC "exec" "Execute a command in a running container" + <> commandDC runDC "help" "Get help on a command" + <> commandDC runEvalAndDC "images" "List images" + <> commandDC runEvalAndDC "kill" "Kill containers" + <> commandDC runEvalAndDC "logs" "View output from containers" + <> commandDC runEvalAndDC "pause" "Pause services" + <> commandDC runEvalAndDC "port" "Print the public port for a port binding" + <> commandDC runEvalAndDC "ps" "List containers" + <> commandDC runBuildAndDC "pull" "Pull service images" + <> commandDC runBuildAndDC "push" "Push service images" + <> commandDC runBuildAndDC "restart" "Restart services" + <> commandDC runEvalAndDC "rm" "Remove stopped containers" + <> commandDC runBuildAndDC "run" "Run a one-off command" + <> commandDC runBuildAndDC "scale" "Set number of containers for a service" + <> commandDC runBuildAndDC "start" "Start services" + <> commandDC runEvalAndDC "stop" "Stop services" + <> commandDC runEvalAndDC "top" "Display the running processes" + <> commandDC runEvalAndDC "unpause" "Unpause services" + <> commandDC runBuildAndDC "up" "Create and start containers" + <> commandDC runDC "version" "Show the Docker-Compose version information" <> metavar "DOCKER-COMPOSE-COMMAND" <> commandGroup "Docker Compose Commands:" ) - -dcParser - :: Text - -> DockerComposeCmd - -> Text - -> Mod CommandFields (CommonOptions -> IO ()) -dcParser cmdStr cmd help = - command - (T.unpack cmdStr) - (info - (runDockerCompose <$> pure cmd <*> parseDockerComposeArgs) - (progDesc (T.unpack help) <> fullDesc <> forwardOptions)) - - parseAll :: Parser (IO ()) parseAll = flip ($) <$> parseOptions <*> parseCommand - parseDockerComposeArgs :: Parser DockerComposeArgs parseDockerComposeArgs = DockerComposeArgs <$> many (argument (T.pack <$> str) (metavar "DOCKER-COMPOSE ARGS...")) +commandDC + :: (Text -> DockerComposeArgs -> CommonOptions -> IO ()) + -> Text + -> Text + -> Mod CommandFields (CommonOptions -> IO ()) +commandDC run cmdStr help = + command + (T.unpack cmdStr) + (info + (run cmdStr <$> parseDockerComposeArgs) + (progDesc (T.unpack help) <> fullDesc <> forwardOptions)) -shouldEval :: DockerComposeCmd -> Bool -shouldEval Up = True -shouldEval Down = True - - -runDockerCompose :: DockerComposeCmd -> DockerComposeArgs -> CommonOptions -> IO () -runDockerCompose cmd args opts = T.putStrLn (show cmd) - - -runCat :: CommonOptions -> IO () -runCat (CommonOptions files pkgs) = do - T.putStrLn "Running cat ... TODO" - T.putStrLn (modulesNixExpr files) - - -runRepl :: CommonOptions -> IO () -runRepl opts = T.putStrLn "Running repl ... TODO" - - -runExec :: CommonOptions -> IO () -runExec opts = T.putStrLn "Running exec ... TODO" - +-------------------------------------------------------------------------------- modulesNixExpr :: NonEmpty FilePath -> Text modulesNixExpr = @@ -189,8 +128,38 @@ modulesNixExpr = wrapList s = "[ " <> s <> " ]" +-------------------------------------------------------------------------------- + +runDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () +runDC cmd (DockerComposeArgs args) opts = + T.putStrLn $ "TODO: docker-compose " <> cmd <> " " <> T.unwords args + +runBuildAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () +runBuildAndDC cmd dopts opts = do + T.putStrLn "TODO: build" + runDC cmd dopts opts + +runEvalAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () +runEvalAndDC cmd dopts opts = do + T.putStrLn "TODO: eval" + runDC cmd dopts opts + +runCat :: CommonOptions -> IO () +runCat (CommonOptions files pkgs) = do + T.putStrLn "Running cat ... TODO" + T.putStrLn (modulesNixExpr files) + +runRepl :: CommonOptions -> IO () +runRepl opts = + T.putStrLn "Running repl ... TODO" + +runExec :: CommonOptions -> IO () +runExec opts = + T.putStrLn "Running exec ... TODO" main :: IO () main = (join . execParser) (info (parseAll <**> helper) fullDesc) + where + execParser = customExecParser (prefs showHelpOnEmpty) From 77eadf4c412e17b4927ffd2c6fa120646ccf22f8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 15 Jun 2019 22:07:20 +0200 Subject: [PATCH 07/27] Build Haskell arion instead --- arion-compose.cabal | 11 +++++++++-- nix/overlay.nix | 3 ++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/arion-compose.cabal b/arion-compose.cabal index eb631c0..c7fcef4 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -1,4 +1,4 @@ -cabal-version: 2.2 +cabal-version: 2.4 name: arion-compose version: 0.1.0.0 @@ -15,6 +15,13 @@ maintainer: robert@hercules-ci.com extra-source-files: CHANGELOG.md, README.asciidoc write-ghc-enviroment-files: never +data-files: nix/*.nix + , nix/modules/composition/*.nix + , nix/modules/nixos/*.nix + , nix/modules/service/*.nix + +-- all data is verbatim from some sources +data-dir: src common deps build-depends: base ^>=4.12.0.0 @@ -45,7 +52,7 @@ executable arion test-suite arion-unit-tests import: deps type: exitcode-stdio-1.0 - main-is: Main.hs + main-is: TestMain.hs -- other-modules: -- other-extensions: build-depends: arion-compose diff --git a/nix/overlay.nix b/nix/overlay.nix index 30a6b7a..97c319f 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -3,7 +3,8 @@ let inherit (self.arion-project) haskellPkgs; in { - arion = super.callPackage ../arion.nix {}; + arion-v0 = super.callPackage ../arion.nix {}; + arion = super.haskell.lib.justStaticExecutables haskellPkgs.arion-compose; tests = super.callPackage ../tests {}; doc = super.callPackage ../doc {}; From 6882a92e560f4ab8b523e418323bce2caf6b5436 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 23 Jun 2019 21:27:13 +0200 Subject: [PATCH 08/27] Make arion cat work --- CHANGELOG.md | 1 + arion-compose.cabal | 13 ++- arion.nix | 1 + nix/overlay.nix | 14 ++- src/arion-image/Dockerfile | 6 +- src/arion-image/passwd | 2 - src/arion-image/tarball/bin/sh | 1 - src/arion-image/tarball/usr/bin/env | 1 - src/haskell/exe/Main.hs | 35 +++---- src/haskell/lib/Arion/Nix.hs | 91 +++++++++++++++++++ .../modules/composition/docker-compose.nix | 10 +- 11 files changed, 143 insertions(+), 32 deletions(-) delete mode 100644 src/arion-image/passwd delete mode 120000 src/arion-image/tarball/bin/sh delete mode 120000 src/arion-image/tarball/usr/bin/env create mode 100644 src/haskell/lib/Arion/Nix.hs diff --git a/CHANGELOG.md b/CHANGELOG.md index 405dedc..6c8d48d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,3 +3,4 @@ ## 0.1.0.0 -- YYYY-mm-dd * First version. Released on an unsuspecting world. +* *BREAKING:* useHostStore now uses a proper empty base image, like `scratch`. diff --git a/arion-compose.cabal b/arion-compose.cabal index c7fcef4..b0e56e8 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -19,6 +19,7 @@ data-files: nix/*.nix , nix/modules/composition/*.nix , nix/modules/nixos/*.nix , nix/modules/service/*.nix + , arion-image/Dockerfile -- all data is verbatim from some sources data-dir: src @@ -26,16 +27,19 @@ data-dir: src common deps build-depends: base ^>=4.12.0.0 , aeson + , async + , bytestring + , process + , process-extras , text , protolude library import: deps - -- exposed-modules: - -- other-modules: + exposed-modules: Arion.Nix + other-modules: Paths_arion_compose -- other-extensions: - build-depends: process hs-source-dirs: src/haskell/lib default-language: Haskell2010 @@ -45,7 +49,8 @@ executable arion -- other-modules: -- other-extensions: build-depends: optparse-applicative - , process + , aeson-pretty + , arion-compose hs-source-dirs: src/haskell/exe default-language: Haskell2010 diff --git a/arion.nix b/arion.nix index 9a23bde..2b4f81a 100644 --- a/arion.nix +++ b/arion.nix @@ -3,6 +3,7 @@ }: let + # TODO: Replace by a new expression for the new Haskell main arion = stdenv.mkDerivation { name = "arion"; src = ./src; diff --git a/nix/overlay.nix b/nix/overlay.nix index 97c319f..3ddc2f6 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,10 +1,22 @@ self: super: let inherit (self.arion-project) haskellPkgs; + + srcDir = ../src; # TODO gitignoreSource + whitelist nix and arion-image + eval = import (srcDir + "/nix/eval-composition.nix"); + build = args@{...}: + let composition = eval args; + in composition.config.build.dockerComposeYaml; + hlib = super.haskell.lib; in { + arion-v0 = super.callPackage ../arion.nix {}; - arion = super.haskell.lib.justStaticExecutables haskellPkgs.arion-compose; + arion = hlib.justStaticExecutables (hlib.overrideCabal haskellPkgs.arion-compose (o: { + passthru = o.passthru // { + inherit eval build; + }; + })); tests = super.callPackage ../tests {}; doc = super.callPackage ../doc {}; diff --git a/src/arion-image/Dockerfile b/src/arion-image/Dockerfile index 4aa6448..b4a070a 100644 --- a/src/arion-image/Dockerfile +++ b/src/arion-image/Dockerfile @@ -1,3 +1,5 @@ FROM scratch -COPY passwd /etc/passwd -ADD tarball.tar.gz / +# scratch itself can't be run. + +# This seems like a no-op: +CMD [] diff --git a/src/arion-image/passwd b/src/arion-image/passwd deleted file mode 100644 index 321840c..0000000 --- a/src/arion-image/passwd +++ /dev/null @@ -1,2 +0,0 @@ -root:x:0:0:System administrator:/root:/bin/sh -nobody:x:65534:65534:Unprivileged account (don't use!):/var/empty:/run/current-system/sw/bin/nologin diff --git a/src/arion-image/tarball/bin/sh b/src/arion-image/tarball/bin/sh deleted file mode 120000 index 261b8af..0000000 --- a/src/arion-image/tarball/bin/sh +++ /dev/null @@ -1 +0,0 @@ -/run/system/bin/sh \ No newline at end of file diff --git a/src/arion-image/tarball/usr/bin/env b/src/arion-image/tarball/usr/bin/env deleted file mode 120000 index 5871f63..0000000 --- a/src/arion-image/tarball/usr/bin/env +++ /dev/null @@ -1 +0,0 @@ -/run/system/usr/bin/env \ No newline at end of file diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index fe4f608..a4a0ec6 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -5,15 +5,21 @@ import Protolude hiding (Down) +import Arion.Nix + import Options.Applicative import Control.Applicative +import qualified Data.Aeson.Encode.Pretty import qualified Data.Text as T import qualified Data.Text.IO as T +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Builder as TB import qualified Data.List.NonEmpty as NE import Data.List.NonEmpty (NonEmpty(..)) + import Control.Arrow ((>>>)) data CommonOptions = @@ -46,7 +52,7 @@ parseOptions = do <> metavar "EXPR" <> showDefault <> value "./arion-pkgs.nix" - <> help "Use EXPR to get the Nixpkgs attrset used for bootstrapping \ + <> help "Use Nix expression EXPR to get the Nixpkgs attrset used for bootstrapping \ \and evaluating the configuration." ) pure CommonOptions{..} @@ -111,28 +117,12 @@ commandDC run cmdStr help = (run cmdStr <$> parseDockerComposeArgs) (progDesc (T.unpack help) <> fullDesc <> forwardOptions)) --------------------------------------------------------------------------------- - -modulesNixExpr :: NonEmpty FilePath -> Text -modulesNixExpr = - NE.toList - >>> fmap pathExpr - >>> T.unwords - >>> wrapList - where - pathExpr path | isAbsolute path = "(/. + \"" <> T.pack path <> "\")" - | otherwise = "(./. + \"" <> T.pack path <> "\")" - - isAbsolute ('/':_) = True - isAbsolute _ = False - - wrapList s = "[ " <> s <> " ]" -------------------------------------------------------------------------------- runDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runDC cmd (DockerComposeArgs args) opts = - T.putStrLn $ "TODO: docker-compose " <> cmd <> " " <> T.unwords args + panic $ "TODO: docker-compose " <> cmd <> " " <> T.unwords args runBuildAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runBuildAndDC cmd dopts opts = do @@ -146,8 +136,13 @@ runEvalAndDC cmd dopts opts = do runCat :: CommonOptions -> IO () runCat (CommonOptions files pkgs) = do - T.putStrLn "Running cat ... TODO" - T.putStrLn (modulesNixExpr files) + v <- Arion.Nix.evaluate EvaluationArgs + { evalUid = 0 -- TODO + , evalModules = files + , evalPkgs = pkgs + , evalWorkDir = Nothing + } + T.hPutStrLn stderr (TL.toStrict $ TB.toLazyText $ Data.Aeson.Encode.Pretty.encodePrettyToTextBuilder v) runRepl :: CommonOptions -> IO () runRepl opts = diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs new file mode 100644 index 0000000..f985a90 --- /dev/null +++ b/src/haskell/lib/Arion/Nix.hs @@ -0,0 +1,91 @@ +{-# LANGUAGE OverloadedStrings #-} +module Arion.Nix where + +import Prelude ( ) +import Protolude +import Data.Aeson +import qualified Data.String +import System.Process +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BL +import qualified System.Process.ByteString.Lazy + as PBL +import Paths_arion_compose +import Control.Applicative + +import qualified Data.Text as T +import qualified Data.Text.IO as T + +import qualified Data.List.NonEmpty as NE +import Data.List.NonEmpty ( NonEmpty(..) ) + +import Control.Arrow ( (>>>) ) + +data EvaluationArgs = EvaluationArgs + { evalUid :: Int + , evalModules :: NonEmpty FilePath + , evalPkgs :: Text + , evalWorkDir :: Maybe FilePath + } + +evaluate :: EvaluationArgs -> IO Value +evaluate ea = do + evalComposition <- getDataFileName "nix/eval-composition.nix" + let args = + [ evalComposition + , "--eval" + , "--strict" + , "--read-write-mode" + , "--json" + , "--show-trace" + , "--argstr" + , "uid" + , show $ evalUid ea + , "--arg" + , "modules" + , modulesNixExpr $ evalModules ea + , "--arg" + , "pkgs" + , toS $ evalPkgs ea + , "--attr" + , "config.build.dockerComposeYamlAttrs" + ] + stdin = mempty + procSpec = (proc "nix-instantiate" args) { cwd = evalWorkDir ea } + + -- TODO: lazy IO is tricky. Let's use conduit/pipes instead? + (exitCode, out, err) <- PBL.readCreateProcessWithExitCode procSpec stdin + + -- Stream 'err' + errDone <- async (BL.hPutStr stderr err) + + -- Force 'out' + v <- Protolude.evaluate (eitherDecode out) + + -- Wait for process exit and 'err' printout + wait errDone + + case exitCode of + ExitSuccess -> pass + ExitFailure e -> throwIO $ FatalError "Evaluation failed" -- TODO: don't print this exception in main + + case v of + Right r -> pure r + Left e -> throwIO $ FatalError "Couldn't parse nix-instantiate output" + + +modulesNixExpr :: NonEmpty FilePath -> [Char] +modulesNixExpr = + NE.toList >>> fmap pathExpr >>> Data.String.unwords >>> wrapList + where + pathExpr :: FilePath -> [Char] + pathExpr path | isAbsolute path = "(/. + \"/${" <> toNixStringLiteral path <> "}\")" + | otherwise = "(./. + \"/${" <> toNixStringLiteral path <> "}\")" + + isAbsolute ('/' : _) = True + isAbsolute _ = False + + wrapList s = "[ " <> s <> " ]" + +toNixStringLiteral :: [Char] -> [Char] +toNixStringLiteral = show -- FIXME: custom escaping including '$' diff --git a/src/nix/modules/composition/docker-compose.nix b/src/nix/modules/composition/docker-compose.nix index 95855ff..6734041 100644 --- a/src/nix/modules/composition/docker-compose.nix +++ b/src/nix/modules/composition/docker-compose.nix @@ -19,10 +19,17 @@ in build.dockerComposeYaml = lib.mkOption { type = lib.types.package; description = "A derivation that produces a docker-compose.yaml file for this composition."; + readOnly = true; }; build.dockerComposeYamlText = lib.mkOption { type = lib.types.string; description = "The text of build.dockerComposeYaml."; + readOnly = true; + }; + build.dockerComposeYamlAttrs = lib.mkOption { + type = lib.types.attrsOf lib.types.unspecified; + description = "The text of build.dockerComposeYaml."; + readOnly = true; }; docker-compose.raw = lib.mkOption { type = lib.types.attrs; @@ -45,7 +52,8 @@ in }; config = { build.dockerComposeYaml = pkgs.writeText "docker-compose.yaml" config.build.dockerComposeYamlText; - build.dockerComposeYamlText = builtins.toJSON (config.docker-compose.raw); + build.dockerComposeYamlText = builtins.toJSON (config.build.dockerComposeYamlAttrs); + build.dockerComposeYamlAttrs = config.docker-compose.raw; docker-compose.evaluatedServices = lib.mapAttrs evalService config.docker-compose.services; docker-compose.raw = { From 36e48776cf12396ea66980ea10119926d81be141 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 23 Jun 2019 21:27:44 +0200 Subject: [PATCH 09/27] Cover up the -compose part of the Haskell package --- nix/overlay.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/overlay.nix b/nix/overlay.nix index 3ddc2f6..0915ceb 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -16,6 +16,7 @@ in passthru = o.passthru // { inherit eval build; }; + pname = "arion"; # Cover up the needlessly long Haskell package name })); tests = super.callPackage ../tests {}; doc = super.callPackage ../doc {}; From c2bdf8e0478a4bd3cf63f1baec1986a81861a761 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 23 Jun 2019 21:33:01 +0200 Subject: [PATCH 10/27] Add scripts for hacking --- .gitignore | 2 ++ build | 6 ++++++ cabal.project | 1 + repl | 6 ++++++ run-arion | 15 +++++++++++++++ run-arion-via-nix | 6 ++++++ 6 files changed, 36 insertions(+) create mode 100755 build create mode 100644 cabal.project create mode 100755 repl create mode 100755 run-arion create mode 100755 run-arion-via-nix diff --git a/.gitignore b/.gitignore index c372372..49eb116 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ result-* dist/ dist-newstyle/ +cabal.project.local + diff --git a/build b/build new file mode 100755 index 0000000..b8f345f --- /dev/null +++ b/build @@ -0,0 +1,6 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash + +# Build the Haskell package via cabal, outside Nix + +cabal new-build --write-ghc-environment-files=never diff --git a/cabal.project b/cabal.project new file mode 100644 index 0000000..5356e76 --- /dev/null +++ b/cabal.project @@ -0,0 +1 @@ +packages: . \ No newline at end of file diff --git a/repl b/repl new file mode 100755 index 0000000..ee3c1ba --- /dev/null +++ b/repl @@ -0,0 +1,6 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash + +# A Haskell REPL for hacking on the Haskell code + +cabal new-repl --write-ghc-environment-files=never diff --git a/run-arion b/run-arion new file mode 100755 index 0000000..f5f2c18 --- /dev/null +++ b/run-arion @@ -0,0 +1,15 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash +#!nix-shell ./shell.nix + +# For quick manual testing of a hacked arion + +# NB: Only works inside the project directory + +cabal \ + new-run \ + --write-ghc-environment-files=never \ + :pkg:arion-compose:exe:arion \ + -- \ + "$@" \ + ; diff --git a/run-arion-via-nix b/run-arion-via-nix new file mode 100755 index 0000000..b04cdb3 --- /dev/null +++ b/run-arion-via-nix @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# For manual testing of a hacked arion built via Nix. +# Works when called from outside the project directory. + +exec nix run -f "$(dirname ${BASH_SOURCE[0]})" arion -c arion "$@" From 381c3ec37f8cdf7d15d2201f00f55bbb83643833 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 29 Jul 2019 13:46:49 +0200 Subject: [PATCH 11/27] Fix live-unit-tests --- arion-compose.cabal | 6 ++++++ live-unit-tests | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/arion-compose.cabal b/arion-compose.cabal index b0e56e8..1637b44 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -34,6 +34,9 @@ common deps , text , protolude +flag ghci + default: False + manual: True library import: deps @@ -56,6 +59,9 @@ executable arion test-suite arion-unit-tests import: deps + if flag(ghci) + hs-source-dirs: src/haskell/lib + ghc-options: -Wno-missing-home-modules type: exitcode-stdio-1.0 main-is: TestMain.hs -- other-modules: diff --git a/live-unit-tests b/live-unit-tests index 09e3af9..4f70f27 100755 --- a/live-unit-tests +++ b/live-unit-tests @@ -6,7 +6,7 @@ set -eux -o pipefail cd "$(dirname "${BASH_SOURCE[0]}")" ghcid \ - --command 'ghci -isrc/haskell/exe -isrc/haskell/lib -isrc/haskell/test src/haskell/test/TestMain.hs' \ + --command 'cabal v2-repl arion-compose:arion-unit-tests --flags ghci --write-ghc-environment-files=never' \ --test=Main.main \ --reload=src/haskell \ --restart=hercules-ci-api.cabal \ From 6d6361e7e886a14150a553bc7df966ae527e5163 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 29 Jul 2019 13:49:26 +0200 Subject: [PATCH 12/27] Add --show-trace, eval unit test --- arion-compose.cabal | 6 ++- nix/haskell-overlay.nix | 14 +++++- src/haskell/exe/Main.hs | 22 +++++++-- src/haskell/lib/Arion/Aeson.hs | 20 ++++++++ src/haskell/lib/Arion/Nix.hs | 29 ++++++++--- src/haskell/test/Arion/FooSpec.hs | 11 ----- src/haskell/test/Arion/NixSpec.hs | 49 +++++++++++++++++++ src/haskell/test/Spec.hs | 4 +- .../testdata/Arion/NixSpec/arion-compose.json | 45 +++++++++++++++++ .../testdata/Arion/NixSpec/arion-compose.nix | 12 +++++ 10 files changed, 182 insertions(+), 30 deletions(-) create mode 100644 src/haskell/lib/Arion/Aeson.hs delete mode 100644 src/haskell/test/Arion/FooSpec.hs create mode 100644 src/haskell/test/Arion/NixSpec.hs create mode 100644 src/haskell/testdata/Arion/NixSpec/arion-compose.json create mode 100644 src/haskell/testdata/Arion/NixSpec/arion-compose.nix diff --git a/arion-compose.cabal b/arion-compose.cabal index 1637b44..149839e 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -27,6 +27,7 @@ data-dir: src common deps build-depends: base ^>=4.12.0.0 , aeson + , aeson-pretty , async , bytestring , process @@ -41,6 +42,7 @@ flag ghci library import: deps exposed-modules: Arion.Nix + Arion.Aeson other-modules: Paths_arion_compose -- other-extensions: hs-source-dirs: src/haskell/lib @@ -52,7 +54,6 @@ executable arion -- other-modules: -- other-extensions: build-depends: optparse-applicative - , aeson-pretty , arion-compose hs-source-dirs: src/haskell/exe default-language: Haskell2010 @@ -64,7 +65,8 @@ test-suite arion-unit-tests ghc-options: -Wno-missing-home-modules type: exitcode-stdio-1.0 main-is: TestMain.hs - -- other-modules: + other-modules: Spec + , Arion.NixSpec -- other-extensions: build-depends: arion-compose , hspec diff --git a/nix/haskell-overlay.nix b/nix/haskell-overlay.nix index 9de9363..74e43c4 100644 --- a/nix/haskell-overlay.nix +++ b/nix/haskell-overlay.nix @@ -1,3 +1,13 @@ -self: super: hself: hsuper: { - arion-compose = hself.callCabal2nix "arion-compose" ./.. {}; +self: super: hself: hsuper: +let + inherit (self.haskell.lib) addBuildTools overrideCabal; +in +{ + arion-compose = overrideCabal (addBuildTools (hself.callCabal2nix "arion-compose" ./.. {}) [self.nix]) (o: o // { + preCheck = '' + export NIX_LOG_DIR=$TMPDIR + export NIX_STATE_DIR=$TMPDIR + export NIX_PATH=nixpkgs=${self.path} + ''; + }); } \ No newline at end of file diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index a4a0ec6..ca6242b 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -6,6 +6,7 @@ import Protolude hiding (Down) import Arion.Nix +import Arion.Aeson import Options.Applicative import Control.Applicative @@ -26,6 +27,7 @@ data CommonOptions = CommonOptions { files :: NonEmpty FilePath , pkgs :: Text + , nixArgs :: [Text] } deriving (Show) @@ -54,7 +56,15 @@ parseOptions = do <> value "./arion-pkgs.nix" <> help "Use Nix expression EXPR to get the Nixpkgs attrset used for bootstrapping \ \and evaluating the configuration." ) - pure CommonOptions{..} + showTrace <- flag False True (long "show-trace" + <> help "Causes Nix to print out a stack trace in case of Nix expression evaluation errors.") + -- TODO --option support (https://github.com/pcapriotti/optparse-applicative/issues/284) + userNixArgs <- many (T.pack <$> strOption (long "nix-arg" <> metavar "ARG" <> help "Pass an extra argument to nix. Example: --nix-arg --option --nix-arg substitute --nix-arg false")) + pure $ + let nixArgs = userNixArgs <|> "--show-trace" <$ guard showTrace + in CommonOptions{..} + +textArgument = fmap T.pack . strArgument parseCommand :: Parser (CommonOptions -> IO ()) parseCommand = @@ -135,14 +145,16 @@ runEvalAndDC cmd dopts opts = do runDC cmd dopts opts runCat :: CommonOptions -> IO () -runCat (CommonOptions files pkgs) = do +runCat co = do v <- Arion.Nix.evaluate EvaluationArgs { evalUid = 0 -- TODO - , evalModules = files - , evalPkgs = pkgs + , evalModules = files co + , evalPkgs = pkgs co , evalWorkDir = Nothing + , evalMode = ReadWrite + , evalUserArgs = nixArgs co } - T.hPutStrLn stderr (TL.toStrict $ TB.toLazyText $ Data.Aeson.Encode.Pretty.encodePrettyToTextBuilder v) + T.hPutStrLn stderr (pretty v) runRepl :: CommonOptions -> IO () runRepl opts = diff --git a/src/haskell/lib/Arion/Aeson.hs b/src/haskell/lib/Arion/Aeson.hs new file mode 100644 index 0000000..f36a1c8 --- /dev/null +++ b/src/haskell/lib/Arion/Aeson.hs @@ -0,0 +1,20 @@ +module Arion.Aeson where + +import Data.Aeson +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Aeson.Encode.Pretty +import Data.Aeson.Encode.Pretty ( defConfig + , keyOrder + , confCompare + , confTrailingNewline + ) +import Protolude + +pretty :: ToJSON a => a -> Text +pretty = + TL.toStrict + . TB.toLazyText + . Data.Aeson.Encode.Pretty.encodePrettyToTextBuilder' config + where config = defConfig { confCompare = compare, confTrailingNewline = True } diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index f985a90..d955484 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -21,24 +21,30 @@ import Data.List.NonEmpty ( NonEmpty(..) ) import Control.Arrow ( (>>>) ) +data EvaluationMode = + ReadWrite | ReadOnly + data EvaluationArgs = EvaluationArgs { evalUid :: Int , evalModules :: NonEmpty FilePath , evalPkgs :: Text , evalWorkDir :: Maybe FilePath + , evalMode :: EvaluationMode + , evalUserArgs :: [Text] } evaluate :: EvaluationArgs -> IO Value evaluate ea = do evalComposition <- getDataFileName "nix/eval-composition.nix" - let args = - [ evalComposition - , "--eval" + let commandArgs = + [ "--eval" , "--strict" - , "--read-write-mode" , "--json" - , "--show-trace" - , "--argstr" + , "--attr" + , "config.build.dockerComposeYamlAttrs" + ] + argArgs = + [ "--argstr" , "uid" , show $ evalUid ea , "--arg" @@ -47,9 +53,13 @@ evaluate ea = do , "--arg" , "pkgs" , toS $ evalPkgs ea - , "--attr" - , "config.build.dockerComposeYamlAttrs" ] + args = + [ evalComposition ] + ++ commandArgs + ++ modeArguments (evalMode ea) + ++ argArgs + ++ map toS (evalUserArgs ea) stdin = mempty procSpec = (proc "nix-instantiate" args) { cwd = evalWorkDir ea } @@ -73,6 +83,9 @@ evaluate ea = do Right r -> pure r Left e -> throwIO $ FatalError "Couldn't parse nix-instantiate output" +modeArguments :: EvaluationMode -> [[Char]] +modeArguments ReadWrite = [ "--read-write-mode" ] +modeArguments ReadOnly = [ "--readonly-mode" ] modulesNixExpr :: NonEmpty FilePath -> [Char] modulesNixExpr = diff --git a/src/haskell/test/Arion/FooSpec.hs b/src/haskell/test/Arion/FooSpec.hs deleted file mode 100644 index d4ea5fd..0000000 --- a/src/haskell/test/Arion/FooSpec.hs +++ /dev/null @@ -1,11 +0,0 @@ -module Arion.FooSpec - ( spec - ) -where - -import Test.Hspec -import Test.QuickCheck - -spec :: Spec -spec = do - it "foo" $ property True diff --git a/src/haskell/test/Arion/NixSpec.hs b/src/haskell/test/Arion/NixSpec.hs new file mode 100644 index 0000000..6b2308e --- /dev/null +++ b/src/haskell/test/Arion/NixSpec.hs @@ -0,0 +1,49 @@ +{-# LANGUAGE OverloadedStrings #-} +module Arion.NixSpec + ( spec + ) +where + +import Protolude +import Test.Hspec +import Test.QuickCheck +import qualified Data.List.NonEmpty as NEL +import Arion.Aeson +import Arion.Nix +import qualified Data.Text as T +import qualified Data.Text.IO as T +import qualified Data.Text.Lazy.IO as TL +import qualified Data.Text.Lazy.Builder as TB +import qualified Data.Aeson.Encode.Pretty +import Data.Char (isSpace) + +spec :: Spec +spec = describe "evaluate" $ it "matches an example" $ do + x <- Arion.Nix.evaluate EvaluationArgs + { evalUid = 123 + , evalModules = NEL.fromList + ["src/haskell/testdata/Arion/NixSpec/arion-compose.nix"] + , evalPkgs = "import {}" + , evalWorkDir = Nothing + , evalMode = ReadOnly + , evalUserArgs = ["--show-trace"] + } + let actual = pretty x + expected <- T.readFile "src/haskell/testdata/Arion/NixSpec/arion-compose.json" + censorPaths actual `shouldBe` censorPaths expected + +censorPaths :: Text -> Text +censorPaths x = case T.breakOn "/nix/store/" x of + (prefix, tl) | (tl :: Text) == "" -> prefix + (prefix, tl) -> prefix <> "" <> censorPaths + (T.dropWhile isNixNameChar $ T.drop (T.length "/nix/store/") tl) + +-- | WARNING: THIS IS LIKELY WRONG: DON'T REUSE +isNixNameChar :: Char -> Bool +isNixNameChar c | c >= '0' && c <= '9' = True +isNixNameChar c | c >= 'a' && c <= 'z' = True +isNixNameChar c | c >= 'A' && c <= 'Z' = True +isNixNameChar c | c == '-' = True +isNixNameChar c | c == '.' = True +isNixNameChar c | c == '_' = True -- WRONG? +isNixNameChar c = False -- WRONG? diff --git a/src/haskell/test/Spec.hs b/src/haskell/test/Spec.hs index 73ab5ab..d2da234 100644 --- a/src/haskell/test/Spec.hs +++ b/src/haskell/test/Spec.hs @@ -4,8 +4,8 @@ module Spec where import Test.Hspec -import qualified Arion.FooSpec +import qualified Arion.NixSpec spec :: Spec spec = do - describe "Arion.Foo" Arion.FooSpec.spec + describe "Arion.Nix" Arion.NixSpec.spec diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.json b/src/haskell/testdata/Arion/NixSpec/arion-compose.json new file mode 100644 index 0000000..198ef05 --- /dev/null +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.json @@ -0,0 +1,45 @@ +{ + "services": { + "webserver": { + "build": { + "context": "/nix/store/l6jwin74n93d66ralxzb001c22yjii9x-arion-image" + }, + "command": [ + "/nix/store/b9w61w4g8sqgrm3rid6ca22krslqghb3-nixos-system-unnamed-19.03.173100.e726e8291b2/init" + ], + "environment": { + "NIX_REMOTE": "", + "container": "docker" + }, + "image": "arion-base", + "ports": [ + "8000:80" + ], + "stop_signal": "SIGRTMIN+3", + "sysctls": {}, + "tmpfs": [ + "/run", + "/run/wrappers", + "/tmp:exec,mode=777" + ], + "tty": true, + "volumes": [ + "/sys/fs/cgroup:/sys/fs/cgroup:ro", + "/nix/store:/nix/store", + "/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system" + ] + } + }, + "version": "3.4", + "x-arion": { + "images": [], + "serviceInfo": { + "webserver": { + "defaultExec": [ + "/run/current-system/sw/bin/bash", + "-l" + ] + } + } + } +} diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.nix b/src/haskell/testdata/Arion/NixSpec/arion-compose.nix new file mode 100644 index 0000000..2ed625c --- /dev/null +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.nix @@ -0,0 +1,12 @@ +{ + docker-compose.services.webserver = { pkgs, ... }: { + nixos.useSystemd = true; + nixos.configuration.boot.tmpOnTmpfs = true; + nixos.configuration.services.nginx.enable = true; + nixos.configuration.services.nginx.virtualHosts.localhost.root = "${pkgs.nix.doc}/share/doc/nix/manual"; + service.useHostStore = true; + service.ports = [ + "8000:80" # host:container + ]; + }; +} From 81887ba633795a75b2b3d99f3b37b1dac84935ad Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 27 Sep 2019 12:36:23 +0200 Subject: [PATCH 13/27] cat: Write to stdout not stderr --- src/haskell/exe/Main.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index ca6242b..cc30bac 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -154,7 +154,7 @@ runCat co = do , evalMode = ReadWrite , evalUserArgs = nixArgs co } - T.hPutStrLn stderr (pretty v) + T.hPutStrLn stdout (pretty v) runRepl :: CommonOptions -> IO () runRepl opts = From 44df36673c713d8cd8cd24d0a33a2f711e0566ab Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 27 Sep 2019 12:39:07 +0200 Subject: [PATCH 14/27] Rename --- src/haskell/exe/Main.hs | 2 +- src/haskell/lib/Arion/Nix.hs | 4 ++-- src/haskell/test/Arion/NixSpec.hs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index cc30bac..c625bc3 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -146,7 +146,7 @@ runEvalAndDC cmd dopts opts = do runCat :: CommonOptions -> IO () runCat co = do - v <- Arion.Nix.evaluate EvaluationArgs + v <- Arion.Nix.evaluateComposition EvaluationArgs { evalUid = 0 -- TODO , evalModules = files co , evalPkgs = pkgs co diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index d955484..609f38f 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -33,8 +33,8 @@ data EvaluationArgs = EvaluationArgs , evalUserArgs :: [Text] } -evaluate :: EvaluationArgs -> IO Value -evaluate ea = do +evaluateComposition :: EvaluationArgs -> IO Value +evaluateComposition ea = do evalComposition <- getDataFileName "nix/eval-composition.nix" let commandArgs = [ "--eval" diff --git a/src/haskell/test/Arion/NixSpec.hs b/src/haskell/test/Arion/NixSpec.hs index 6b2308e..3d07cb4 100644 --- a/src/haskell/test/Arion/NixSpec.hs +++ b/src/haskell/test/Arion/NixSpec.hs @@ -18,8 +18,8 @@ import qualified Data.Aeson.Encode.Pretty import Data.Char (isSpace) spec :: Spec -spec = describe "evaluate" $ it "matches an example" $ do - x <- Arion.Nix.evaluate EvaluationArgs +spec = describe "evaluateComposition" $ it "matches an example" $ do + x <- Arion.Nix.evaluateComposition EvaluationArgs { evalUid = 123 , evalModules = NEL.fromList ["src/haskell/testdata/Arion/NixSpec/arion-compose.nix"] From c0e995043ab6a68a3403f093f927b3135dff0615 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 27 Sep 2019 12:44:24 +0200 Subject: [PATCH 15/27] Fix unit tests to reflect updated master --- src/haskell/testdata/Arion/NixSpec/arion-compose.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.json b/src/haskell/testdata/Arion/NixSpec/arion-compose.json index 198ef05..33b1f63 100644 --- a/src/haskell/testdata/Arion/NixSpec/arion-compose.json +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.json @@ -9,6 +9,7 @@ ], "environment": { "NIX_REMOTE": "", + "PATH": "/usr/bin:/run/current-system/sw/bin/", "container": "docker" }, "image": "arion-base", @@ -25,8 +26,8 @@ "tty": true, "volumes": [ "/sys/fs/cgroup:/sys/fs/cgroup:ro", - "/nix/store:/nix/store", - "/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system" + "/nix/store:/nix/store:ro", + "/nix/store/pssdmhzjnhflawv7rwk1yw39350iv40g-container-system-env:/run/system:ro" ] } }, From b9488b7f49aed8cb989f8720427c807d5b060d45 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 27 Sep 2019 21:01:57 +0200 Subject: [PATCH 16/27] Make some commands work --- arion-compose.cabal | 3 + nix/overlay.nix | 1 + src/haskell/exe/Main.hs | 43 +++++++---- src/haskell/lib/Arion/DockerCompose.hs | 51 ++++++++++++++ src/haskell/lib/Arion/Nix.hs | 98 ++++++++++++++++++++++---- 5 files changed, 168 insertions(+), 28 deletions(-) create mode 100644 src/haskell/lib/Arion/DockerCompose.hs diff --git a/arion-compose.cabal b/arion-compose.cabal index 149839e..d413515 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -30,8 +30,10 @@ common deps , aeson-pretty , async , bytestring + , directory , process , process-extras + , temporary , text , protolude @@ -43,6 +45,7 @@ library import: deps exposed-modules: Arion.Nix Arion.Aeson + Arion.DockerCompose other-modules: Paths_arion_compose -- other-extensions: hs-source-dirs: src/haskell/lib diff --git a/nix/overlay.nix b/nix/overlay.nix index 0915ceb..3105187 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -28,6 +28,7 @@ in buildInputs = [ haskellPkgs.cabal-install haskellPkgs.ghcid + super.docker-compose ]; }; }; diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index c625bc3..b5910d0 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -7,6 +7,7 @@ import Protolude hiding (Down) import Arion.Nix import Arion.Aeson +import qualified Arion.DockerCompose as DockerCompose import Options.Applicative import Control.Applicative @@ -131,29 +132,43 @@ commandDC run cmdStr help = -------------------------------------------------------------------------------- runDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () -runDC cmd (DockerComposeArgs args) opts = - panic $ "TODO: docker-compose " <> cmd <> " " <> T.unwords args +runDC cmd (DockerComposeArgs args) opts = do + DockerCompose.run DockerCompose.Args + { files = [] + , otherArgs = [cmd] ++ args + } runBuildAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runBuildAndDC cmd dopts opts = do - T.putStrLn "TODO: build" - runDC cmd dopts opts + ea <- defaultEvaluationArgs opts + Arion.Nix.withBuiltComposition ea $ \path -> + DockerCompose.run DockerCompose.Args + { files = [path] + , otherArgs = [cmd] ++ unDockerComposeArgs dopts + } runEvalAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runEvalAndDC cmd dopts opts = do - T.putStrLn "TODO: eval" - runDC cmd dopts opts + ea <- defaultEvaluationArgs opts + Arion.Nix.withEvaluatedComposition ea $ \path -> + DockerCompose.run DockerCompose.Args + { files = [path] + , otherArgs = [cmd] ++ unDockerComposeArgs dopts + } + +defaultEvaluationArgs :: CommonOptions -> IO EvaluationArgs +defaultEvaluationArgs co = pure EvaluationArgs + { evalUid = 0 -- TODO + , evalModules = files co + , evalPkgs = pkgs co + , evalWorkDir = Nothing + , evalMode = ReadWrite + , evalUserArgs = nixArgs co + } runCat :: CommonOptions -> IO () runCat co = do - v <- Arion.Nix.evaluateComposition EvaluationArgs - { evalUid = 0 -- TODO - , evalModules = files co - , evalPkgs = pkgs co - , evalWorkDir = Nothing - , evalMode = ReadWrite - , evalUserArgs = nixArgs co - } + v <- Arion.Nix.evaluateComposition =<< defaultEvaluationArgs co T.hPutStrLn stdout (pretty v) runRepl :: CommonOptions -> IO () diff --git a/src/haskell/lib/Arion/DockerCompose.hs b/src/haskell/lib/Arion/DockerCompose.hs new file mode 100644 index 0000000..804f95a --- /dev/null +++ b/src/haskell/lib/Arion/DockerCompose.hs @@ -0,0 +1,51 @@ +{-# LANGUAGE OverloadedStrings #-} +module Arion.DockerCompose where + +import Prelude ( ) +import Protolude +import Arion.Aeson ( pretty ) +import Data.Aeson +import qualified Data.String +import System.Process +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as BL +import qualified System.Process.ByteString.Lazy + as PBL +import Paths_arion_compose +import Control.Applicative + +import qualified Data.Text as T +import qualified Data.Text.IO as T + +import qualified Data.List.NonEmpty as NE +import Data.List.NonEmpty ( NonEmpty(..) ) + +import Control.Arrow ( (>>>) ) +import System.IO.Temp ( withTempFile ) +import System.IO ( hClose ) + +data Args = Args + { files :: [FilePath] + , otherArgs :: [Text] + } + +run :: Args -> IO () +run args = do + let fileArgs = files args >>= \f -> ["--file", f] + allArgs = fileArgs ++ map toS (otherArgs args) + + procSpec = proc "docker-compose" allArgs + + -- hPutStrLn stderr ("Running docker-compose with " <> show allArgs :: Text) + + withCreateProcess procSpec $ \_in _out _err procHandle -> do + + -- Wait for process exit and 'err' printout + exitCode <- waitForProcess procHandle + + case exitCode of + ExitSuccess -> pass + ExitFailure 1 -> exitFailure + e@ExitFailure {} -> do + throwIO $ FatalError $ "docker-compose failed with " <> show exitCode + exitWith e diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index 609f38f..dd06692 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -1,10 +1,18 @@ {-# LANGUAGE OverloadedStrings #-} -module Arion.Nix where +module Arion.Nix + ( evaluateComposition + , withEvaluatedComposition + , withBuiltComposition + , EvaluationArgs(..) + , EvaluationMode(..) + ) where import Prelude ( ) import Protolude +import Arion.Aeson ( pretty ) import Data.Aeson import qualified Data.String +import qualified System.Directory as Directory import System.Process import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL @@ -20,6 +28,8 @@ import qualified Data.List.NonEmpty as NE import Data.List.NonEmpty ( NonEmpty(..) ) import Control.Arrow ( (>>>) ) +import System.IO.Temp ( withTempFile ) +import System.IO ( hClose ) data EvaluationMode = ReadWrite | ReadOnly @@ -35,7 +45,7 @@ data EvaluationArgs = EvaluationArgs evaluateComposition :: EvaluationArgs -> IO Value evaluateComposition ea = do - evalComposition <- getDataFileName "nix/eval-composition.nix" + evalComposition <- getEvalCompositionFile let commandArgs = [ "--eval" , "--strict" @@ -43,22 +53,11 @@ evaluateComposition ea = do , "--attr" , "config.build.dockerComposeYamlAttrs" ] - argArgs = - [ "--argstr" - , "uid" - , show $ evalUid ea - , "--arg" - , "modules" - , modulesNixExpr $ evalModules ea - , "--arg" - , "pkgs" - , toS $ evalPkgs ea - ] args = [ evalComposition ] ++ commandArgs ++ modeArguments (evalMode ea) - ++ argArgs + ++ argArgs ea ++ map toS (evalUserArgs ea) stdin = mempty procSpec = (proc "nix-instantiate" args) { cwd = evalWorkDir ea } @@ -83,6 +82,77 @@ evaluateComposition ea = do Right r -> pure r Left e -> throwIO $ FatalError "Couldn't parse nix-instantiate output" +-- | Run with docker-compose.yaml tmpfile +withEvaluatedComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r +withEvaluatedComposition ea f = do + v <- evaluateComposition ea + withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path handle -> do + T.hPutStrLn handle (pretty v) + hClose handle + f path + + +buildComposition :: FilePath -> EvaluationArgs -> IO () +buildComposition outLink ea = do + evalComposition <- getEvalCompositionFile + let commandArgs = + [ "--attr" + , "config.build.dockerComposeYaml" + , "--out-link" + , outLink + ] + args = + [ evalComposition ] + ++ commandArgs + ++ argArgs ea + ++ map toS (evalUserArgs ea) + stdin = mempty + procSpec = (proc "nix-build" args) { cwd = evalWorkDir ea } + + -- TODO: lazy IO is tricky. Let's use conduit/pipes instead? + (exitCode, out, err) <- PBL.readCreateProcessWithExitCode procSpec stdin + + -- Stream 'err' + errDone <- async (BL.hPutStr stderr err) + + -- Force 'out' + -- TODO: use it? + _v <- Protolude.evaluate out + + -- Wait for process exit and 'err' printout + wait errDone + + case exitCode of + ExitSuccess -> pass + ExitFailure e -> throwIO $ FatalError "Build failed" -- TODO: don't print this exception in main + +-- | Do something with a docker-compose.yaml. +withBuiltComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r +withBuiltComposition ea f = do + withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path handle -> do + hClose handle + -- Known problem: kills atomicity of withTempFile; won't fix because we should manage gc roots, + -- impl of which will probably avoid this "problem". It seems unlikely to cause issues. + Directory.removeFile path + buildComposition path ea + f path + +argArgs :: EvaluationArgs -> [[Char]] +argArgs ea = + [ "--argstr" + , "uid" + , show $ evalUid ea + , "--arg" + , "modules" + , modulesNixExpr $ evalModules ea + , "--arg" + , "pkgs" + , toS $ evalPkgs ea + ] + +getEvalCompositionFile :: IO FilePath +getEvalCompositionFile = getDataFileName "nix/eval-composition.nix" + modeArguments :: EvaluationMode -> [[Char]] modeArguments ReadWrite = [ "--read-write-mode" ] modeArguments ReadOnly = [ "--readonly-mode" ] From fcf270c80c2a7e47a020f67ab060a0e1bacd2b7c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 27 Sep 2019 21:45:06 +0200 Subject: [PATCH 17/27] Make arion repl work --- src/haskell/exe/Main.hs | 20 +++++++++++++++----- src/haskell/lib/Arion/DockerCompose.hs | 1 - src/haskell/lib/Arion/Nix.hs | 23 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index b5910d0..325c546 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -70,9 +70,9 @@ textArgument = fmap T.pack . strArgument parseCommand :: Parser (CommonOptions -> IO ()) parseCommand = hsubparser - ( command "cat" (info (pure runCat) (progDesc "TODO: cat doc" <> fullDesc)) - <> command "repl" (info (pure runRepl) (progDesc "TODO: repl doc" <> fullDesc)) - <> command "exec" (info (pure runExec) (progDesc "TODO: exec doc" <> fullDesc)) + ( command "cat" (info (pure runCat) (progDesc "Spit out the docker compose file as JSON" <> fullDesc)) + <> command "repl" (info (pure runRepl) (progDesc "Start a nix repl for the whole composition" <> fullDesc)) + -- <> command "exec" (info (pure runExec) (progDesc "TODO: exec doc" <> fullDesc)) ) <|> hsubparser @@ -172,8 +172,18 @@ runCat co = do T.hPutStrLn stdout (pretty v) runRepl :: CommonOptions -> IO () -runRepl opts = - T.putStrLn "Running repl ... TODO" +runRepl co = do + putErrText + "Launching a repl for you. To get started:\n\ + \\n\ + \To see deployment-wide configuration\n\ + \ type config. and hit TAB\n\ + \To see the services\n\ + \ type config.docker-compose.evaluatedServices TAB or ENTER\n\ + \To bring the top-level Nixpkgs attributes into scope\n\ + \ type :a (config._module.args.pkgs) // { inherit config; }\n\ + \" + Arion.Nix.replForComposition =<< defaultEvaluationArgs co runExec :: CommonOptions -> IO () runExec opts = diff --git a/src/haskell/lib/Arion/DockerCompose.hs b/src/haskell/lib/Arion/DockerCompose.hs index 804f95a..898eb9f 100644 --- a/src/haskell/lib/Arion/DockerCompose.hs +++ b/src/haskell/lib/Arion/DockerCompose.hs @@ -40,7 +40,6 @@ run args = do withCreateProcess procSpec $ \_in _out _err procHandle -> do - -- Wait for process exit and 'err' printout exitCode <- waitForProcess procHandle case exitCode of diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index dd06692..6eb40d9 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -2,7 +2,9 @@ module Arion.Nix ( evaluateComposition , withEvaluatedComposition + , buildComposition , withBuiltComposition + , replForComposition , EvaluationArgs(..) , EvaluationMode(..) ) where @@ -137,6 +139,27 @@ withBuiltComposition ea f = do buildComposition path ea f path + +replForComposition :: EvaluationArgs -> IO () +replForComposition ea = do + evalComposition <- getEvalCompositionFile + let args = + [ "repl", evalComposition ] + ++ argArgs ea + ++ map toS (evalUserArgs ea) + procSpec = (proc "nix" args) { cwd = evalWorkDir ea } + + withCreateProcess procSpec $ \_in _out _err procHandle -> do + + exitCode <- waitForProcess procHandle + + case exitCode of + ExitSuccess -> pass + ExitFailure 1 -> exitFailure + e@ExitFailure {} -> do + throwIO $ FatalError $ "nix repl failed with " <> show exitCode + exitWith e + argArgs :: EvaluationArgs -> [[Char]] argArgs ea = [ "--argstr" From 1fe10c076d4f3d149cbcbf82d87f5927c40704dc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 27 Sep 2019 23:59:08 +0200 Subject: [PATCH 18/27] Implement image loading, use it instead of arion-base --- arion-compose.cabal | 3 + nix/nixpkgs.nix | 4 +- src/haskell/exe/Main.hs | 4 +- src/haskell/lib/Arion/Aeson.hs | 9 +++ src/haskell/lib/Arion/Images.hs | 60 +++++++++++++++++++ src/haskell/test/Arion/NixSpec.hs | 19 +++++- .../testdata/Arion/NixSpec/arion-compose.json | 13 ++-- src/nix/modules/service/host-store.nix | 11 +++- 8 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 src/haskell/lib/Arion/Images.hs diff --git a/arion-compose.cabal b/arion-compose.cabal index d413515..ade58de 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -31,6 +31,8 @@ common deps , async , bytestring , directory + , lens + , lens-aeson , process , process-extras , temporary @@ -46,6 +48,7 @@ library exposed-modules: Arion.Nix Arion.Aeson Arion.DockerCompose + Arion.Images other-modules: Paths_arion_compose -- other-extensions: hs-source-dirs: src/haskell/lib diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix index 4c64290..662487c 100644 --- a/nix/nixpkgs.nix +++ b/nix/nixpkgs.nix @@ -1,5 +1,5 @@ # to update: $ nix-prefetch-url --unpack url builtins.fetchTarball { - url = "https://github.com/NixOS/nixpkgs/archive/be445a9074f139d63e704fa82610d25456562c3d.tar.gz"; - sha256 = "15dc7gdspimavcwyw9nif4s59v79gk18rwsafylffs9m1ld2dxwa"; + url = "https://github.com/NixOS/nixpkgs/archive/bd5e8f35c2e9d1ddc9cd2fea7a23563336d54acb.tar.gz"; + sha256 = "1wnzqqijrwf797nb234q10zb1h7086njradkkrx3a15b303grsw4"; } diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index 325c546..f7c4035 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -7,6 +7,7 @@ import Protolude hiding (Down) import Arion.Nix import Arion.Aeson +import Arion.Images (loadImages) import qualified Arion.DockerCompose as DockerCompose import Options.Applicative @@ -141,7 +142,8 @@ runDC cmd (DockerComposeArgs args) opts = do runBuildAndDC :: Text -> DockerComposeArgs -> CommonOptions -> IO () runBuildAndDC cmd dopts opts = do ea <- defaultEvaluationArgs opts - Arion.Nix.withBuiltComposition ea $ \path -> + Arion.Nix.withBuiltComposition ea $ \path -> do + loadImages path DockerCompose.run DockerCompose.Args { files = [path] , otherArgs = [cmd] ++ unDockerComposeArgs dopts diff --git a/src/haskell/lib/Arion/Aeson.hs b/src/haskell/lib/Arion/Aeson.hs index f36a1c8..dd3ae12 100644 --- a/src/haskell/lib/Arion/Aeson.hs +++ b/src/haskell/lib/Arion/Aeson.hs @@ -1,6 +1,8 @@ module Arion.Aeson where +import Prelude () import Data.Aeson +import qualified Data.ByteString.Lazy as BL import qualified Data.Text.Lazy as TL import qualified Data.Text.Lazy.IO as TL import qualified Data.Text.Lazy.Builder as TB @@ -18,3 +20,10 @@ pretty = . TB.toLazyText . Data.Aeson.Encode.Pretty.encodePrettyToTextBuilder' config where config = defConfig { confCompare = compare, confTrailingNewline = True } + +decodeFile :: FromJSON a => FilePath -> IO a +decodeFile fp = do + b <- BL.readFile fp + case eitherDecode b of + Left e -> panic (toS e) + Right v -> pure v diff --git a/src/haskell/lib/Arion/Images.hs b/src/haskell/lib/Arion/Images.hs new file mode 100644 index 0000000..cf52e22 --- /dev/null +++ b/src/haskell/lib/Arion/Images.hs @@ -0,0 +1,60 @@ +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE OverloadedStrings #-} +module Arion.Images + ( loadImages + ) where + +import Prelude() +import Protolude hiding (to) + +import qualified Data.Aeson as Aeson +import Arion.Aeson (decodeFile) +import qualified Data.ByteString as BS +import qualified System.Process as Process + +import Control.Lens +import Data.Aeson.Lens +import Data.String +import System.IO (withFile, IOMode(ReadMode)) + + +data Image = Image + { image :: Text -- ^ file path + , imageName :: Text + , imageTag :: Text + } deriving (Generic, Aeson.ToJSON, Aeson.FromJSON, Show) + +type TaggedImage = Text + +loadImages :: FilePath -> IO () +loadImages fp = do + + v <- decodeFile fp + + loaded <- dockerImages + + let + images :: [Image] + images = (v :: Aeson.Value) ^.. key "x-arion" . key "images" . _Array . traverse . _JSON + + isNew i = (imageName i <> ":" <> imageTag i) `notElem` loaded + + traverse_ loadImage . map (toS . image) . filter isNew $ images + +loadImage :: FilePath -> IO () +loadImage imgPath = withFile (imgPath) ReadMode $ \fileHandle -> do + let procSpec = (Process.proc "docker" [ "load" ]) { + Process.std_in = Process.UseHandle fileHandle + } + Process.withCreateProcess procSpec $ \_in _out _err procHandle -> do + e <- Process.waitForProcess procHandle + case e of + ExitSuccess -> pass + ExitFailure code -> panic $ "docker load (" <> show code <> ") failed for " <> toS imgPath + + +dockerImages :: IO [TaggedImage] +dockerImages = do + let procSpec = Process.proc "docker" [ "images", "--filter", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}" ] + (map toS . lines) <$> Process.readCreateProcess procSpec "" diff --git a/src/haskell/test/Arion/NixSpec.hs b/src/haskell/test/Arion/NixSpec.hs index 3d07cb4..884a68a 100644 --- a/src/haskell/test/Arion/NixSpec.hs +++ b/src/haskell/test/Arion/NixSpec.hs @@ -32,12 +32,27 @@ spec = describe "evaluateComposition" $ it "matches an example" $ do expected <- T.readFile "src/haskell/testdata/Arion/NixSpec/arion-compose.json" censorPaths actual `shouldBe` censorPaths expected -censorPaths :: Text -> Text -censorPaths x = case T.breakOn "/nix/store/" x of +censorPaths = censorImages . censorStorePaths +--censorPaths = censorStorePaths + +censorStorePaths :: Text -> Text +censorStorePaths x = case T.breakOn "/nix/store/" x of (prefix, tl) | (tl :: Text) == "" -> prefix (prefix, tl) -> prefix <> "" <> censorPaths (T.dropWhile isNixNameChar $ T.drop (T.length "/nix/store/") tl) +-- Probably slow, due to not O(1) <> +censorImages :: Text -> Text +censorImages x = case T.break (\c -> c == ':' || c == '"') x of + (prefix, tl) | tl == "" -> prefix + (prefix, tl) | let imageId = T.take 33 (T.drop 1 tl) + , T.last imageId == '\"' + -- Approximation of nix hash validation + , T.all (\c -> (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z')) (T.take 32 imageId) + -> prefix <> T.take 1 tl <> "" <> censorImages (T.drop 33 tl) + (prefix, tl) -> prefix <> T.take 1 tl <> censorImages (T.drop 1 tl) + + -- | WARNING: THIS IS LIKELY WRONG: DON'T REUSE isNixNameChar :: Char -> Bool isNixNameChar c | c >= '0' && c <= '9' = True diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.json b/src/haskell/testdata/Arion/NixSpec/arion-compose.json index 33b1f63..37a9051 100644 --- a/src/haskell/testdata/Arion/NixSpec/arion-compose.json +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.json @@ -1,9 +1,6 @@ { "services": { "webserver": { - "build": { - "context": "/nix/store/l6jwin74n93d66ralxzb001c22yjii9x-arion-image" - }, "command": [ "/nix/store/b9w61w4g8sqgrm3rid6ca22krslqghb3-nixos-system-unnamed-19.03.173100.e726e8291b2/init" ], @@ -12,7 +9,7 @@ "PATH": "/usr/bin:/run/current-system/sw/bin/", "container": "docker" }, - "image": "arion-base", + "image": "webserver:", "ports": [ "8000:80" ], @@ -33,7 +30,13 @@ }, "version": "3.4", "x-arion": { - "images": [], + "images": [ + { + "image": "", + "imageName": "webserver", + "imageTag": "" + } + ], "serviceInfo": { "webserver": { "defaultExec": [ diff --git a/src/nix/modules/service/host-store.nix b/src/nix/modules/service/host-store.nix index 63a77c1..f3d1c08 100644 --- a/src/nix/modules/service/host-store.nix +++ b/src/nix/modules/service/host-store.nix @@ -29,9 +29,14 @@ in }; }; config = mkIf config.service.useHostStore { - image.nixBuild = false; # no need to build and load - service.image = "arion-base"; - service.build.context = "${../../../arion-image}"; + image.nixBuild = true; + image.contents = [ + (pkgs.runCommand "minimal-contents" {} '' + mkdir -p $out/bin $out/usr/bin + ln -s /run/system/bin/sh $out/bin/sh + ln -s /run/system/usr/bin/env $out/usr/bin/env + '') + ]; service.environment.NIX_REMOTE = lib.optionalString config.service.useHostNixDaemon "daemon"; service.volumes = [ "${config.host.nixStorePrefix}/nix/store:/nix/store${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}" From 77c492fa8652577568f0e78892b6bb5348ed5d9e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 28 Sep 2019 00:42:03 +0200 Subject: [PATCH 19/27] Wrap arion binary PATH+=docker-compose, unset PYTHONPATH --- nix/overlay.nix | 21 +++++++++++++++++++++ tests/arion-test/default.nix | 1 - 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/nix/overlay.nix b/nix/overlay.nix index 3105187..6a41e14 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,6 +1,7 @@ self: super: let inherit (self.arion-project) haskellPkgs; + inherit (super) lib; srcDir = ../src; # TODO gitignoreSource + whitelist nix and arion-image eval = import (srcDir + "/nix/eval-composition.nix"); @@ -13,10 +14,30 @@ in arion-v0 = super.callPackage ../arion.nix {}; arion = hlib.justStaticExecutables (hlib.overrideCabal haskellPkgs.arion-compose (o: { + buildTools = o.buildTools ++ [super.makeWrapper]; passthru = o.passthru // { inherit eval build; }; pname = "arion"; # Cover up the needlessly long Haskell package name + + # PYTHONPATH + # + # We close off the python module search path! + # + # Accepting directories from the environment into the search path + # tends to break things. Docker Compose does not have a plugin + # system as far as I can tell, so I don't expect this to break a + # feature, but rather to make the program more robustly self- + # contained. + + postInstall = ''${o.postInstall or ""} + mkdir -p $out/libexec + mv $out/bin/arion $out/libexec + makeWrapper $out/libexec/arion $out/bin/arion \ + --unset PYTHONPATH \ + --prefix PATH : ${lib.makeBinPath [ self.docker-compose ]} \ + ; + ''; })); tests = super.callPackage ../tests {}; doc = super.callPackage ../doc {}; diff --git a/tests/arion-test/default.nix b/tests/arion-test/default.nix index 1020a1b..e197fb2 100644 --- a/tests/arion-test/default.nix +++ b/tests/arion-test/default.nix @@ -12,7 +12,6 @@ in machine = { pkgs, lib, ... }: { environment.systemPackages = [ pkgs.arion - pkgs.docker-compose ]; virtualisation.docker.enable = true; From 02c0f80b022641e46430db25d86a0eeaa3988ef0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 28 Sep 2019 00:51:16 +0200 Subject: [PATCH 20/27] Implement uid parameter --- arion-compose.cabal | 1 + src/haskell/exe/Main.hs | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/arion-compose.cabal b/arion-compose.cabal index ade58de..40a45be 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -38,6 +38,7 @@ common deps , temporary , text , protolude + , unix flag ghci default: False diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index f7c4035..fb29f89 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -22,9 +22,10 @@ import qualified Data.Text.Lazy.Builder as TB import qualified Data.List.NonEmpty as NE import Data.List.NonEmpty (NonEmpty(..)) - import Control.Arrow ((>>>)) +import System.Posix.User (getRealUserID) + data CommonOptions = CommonOptions { files :: NonEmpty FilePath @@ -159,14 +160,16 @@ runEvalAndDC cmd dopts opts = do } defaultEvaluationArgs :: CommonOptions -> IO EvaluationArgs -defaultEvaluationArgs co = pure EvaluationArgs - { evalUid = 0 -- TODO - , evalModules = files co - , evalPkgs = pkgs co - , evalWorkDir = Nothing - , evalMode = ReadWrite - , evalUserArgs = nixArgs co - } +defaultEvaluationArgs co = do + uid <- getRealUserID + pure EvaluationArgs + { evalUid = fromIntegral uid + , evalModules = files co + , evalPkgs = pkgs co + , evalWorkDir = Nothing + , evalMode = ReadWrite + , evalUserArgs = nixArgs co + } runCat :: CommonOptions -> IO () runCat co = do From 0474544d0b9adbd6516a95e5a409c2bbf559b36a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 28 Sep 2019 14:24:23 +0200 Subject: [PATCH 21/27] Implement defaultExec --- .envrc | 5 ++ arion-compose.cabal | 1 + src/haskell/exe/Main.hs | 83 ++++++++++++++++++++++++++++--- src/haskell/lib/Arion/Images.hs | 1 + src/haskell/lib/Arion/Nix.hs | 1 - src/haskell/lib/Arion/Services.hs | 27 ++++++++++ 6 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 .envrc create mode 100644 src/haskell/lib/Arion/Services.hs diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..0974ec7 --- /dev/null +++ b/.envrc @@ -0,0 +1,5 @@ +eval "$(lorri direnv)" + +# Use system PKI +unset SSL_CERT_FILE +unset NIX_SSL_CERT_FILE diff --git a/arion-compose.cabal b/arion-compose.cabal index 40a45be..77891a7 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -50,6 +50,7 @@ library Arion.Aeson Arion.DockerCompose Arion.Images + Arion.Services other-modules: Paths_arion_compose -- other-extensions: hs-source-dirs: src/haskell/lib diff --git a/src/haskell/exe/Main.hs b/src/haskell/exe/Main.hs index fb29f89..21ec3d3 100644 --- a/src/haskell/exe/Main.hs +++ b/src/haskell/exe/Main.hs @@ -3,15 +3,17 @@ {-# LANGUAGE ApplicativeDo #-} {-# LANGUAGE OverloadedStrings #-} -import Protolude hiding (Down) +import Protolude hiding (Down, option) import Arion.Nix import Arion.Aeson import Arion.Images (loadImages) import qualified Arion.DockerCompose as DockerCompose +import Arion.Services (getDefaultExec) import Options.Applicative import Control.Applicative +import Control.Monad.Fail import qualified Data.Aeson.Encode.Pretty import qualified Data.Text as T @@ -74,7 +76,7 @@ parseCommand = hsubparser ( command "cat" (info (pure runCat) (progDesc "Spit out the docker compose file as JSON" <> fullDesc)) <> command "repl" (info (pure runRepl) (progDesc "Start a nix repl for the whole composition" <> fullDesc)) - -- <> command "exec" (info (pure runExec) (progDesc "TODO: exec doc" <> fullDesc)) + <> command "exec" (info (parseExecCommand) (progDesc "Execute a command in a running container" <> fullDesc)) ) <|> hsubparser @@ -84,7 +86,6 @@ parseCommand = <> commandDC runBuildAndDC "create" "Create services" <> commandDC runEvalAndDC "down" "Stop and remove containers, networks, images, and volumes" <> commandDC runEvalAndDC "events" "Receive real time events from containers" - <> commandDC runEvalAndDC "exec" "Execute a command in a running container" <> commandDC runDC "help" "Get help on a command" <> commandDC runEvalAndDC "images" "List images" <> commandDC runEvalAndDC "kill" "Kill containers" @@ -190,9 +191,79 @@ runRepl co = do \" Arion.Nix.replForComposition =<< defaultEvaluationArgs co -runExec :: CommonOptions -> IO () -runExec opts = - T.putStrLn "Running exec ... TODO" +detachFlag :: Parser Bool +detachFlag = flag False True (long "detach" <> short 'd' <> help "Detached mode: Run command in the background.") + +privilegedFlag :: Parser Bool +privilegedFlag = flag False True (long "privileged" <> help "Give extended privileges to the process.") + +userOption :: Parser Text +userOption = strOption (long "user" <> short 'u' <> help "Run the command as this user.") + +noTTYFlag :: Parser Bool +noTTYFlag = flag False True (short 'T' <> help "Disable pseudo-tty allocation. By default `exec` allocates a TTY.") + +indexOption :: Parser Int +indexOption = option + (auto >>= \i -> i <$ unless (i >= 1) (fail "container index must be >= 1")) + (long "index" <> value 1 <> help "Index of the container if there are multiple instances of a service.") + +envOption :: Parser (Text, Text) +envOption = option (auto >>= spl) (long "env" <> short 'e' <> help "Set environment variables (can be used multiple times, not supported in Docker API < 1.25)") + where spl s = case T.break (== '=') s of + (_, "") -> fail "--env parameter needs to combine key and value with = sign" + (k, ev) -> pure (k, T.drop 1 ev) + +workdirOption :: Parser Text +workdirOption = strOption (long "workdir" <> short 'w' <> metavar "DIR" <> help "Working directory in which to start the command in the container.") + +parseExecCommand :: Parser (CommonOptions -> IO ()) +parseExecCommand = runExec + <$> detachFlag + <*> privilegedFlag + <*> optional userOption + <*> noTTYFlag + <*> indexOption + <*> many envOption + <*> optional workdirOption + <*> textArgument (metavar "SERVICE") + <*> orEmpty' ( + (:) <$> argument (T.pack <$> str) (metavar "COMMAND") + <*> many (argument (T.pack <$> str) (metavar "ARG")) + ) + +orEmpty' :: (Alternative f, Monoid a) => f a -> f a +orEmpty' m = fromMaybe mempty <$> optional m + +runExec :: Bool -> Bool -> Maybe Text -> Bool -> Int -> [(Text, Text)] -> Maybe Text -> Text -> [Text] -> CommonOptions -> IO () +runExec detach privileged user noTTY index envs workDir service commandAndArgs opts = do + putErrText $ "Service: " <> service + + ea <- defaultEvaluationArgs opts + Arion.Nix.withEvaluatedComposition ea $ \path -> do + commandAndArgs'' <- case commandAndArgs of + [] -> getDefaultExec path service + x -> pure x + let commandAndArgs' = case commandAndArgs'' of + [] -> ["/bin/sh"] + x -> x + + let args = concat + [ ["exec"] + , ("--detach" <$ guard detach :: [Text]) + , "--privileged" <$ guard privileged + , "-T" <$ guard noTTY + , (\(k, v) -> ["--env", k <> "=" <> v]) =<< envs + , join $ toList (user <&> \u -> ["--user", u]) + , ["--index", show index] + , join $ toList (workDir <&> \w -> ["--workdir", w]) + , [service] + , commandAndArgs' + ] + DockerCompose.run DockerCompose.Args + { files = [path] + , otherArgs = args + } main :: IO () main = diff --git a/src/haskell/lib/Arion/Images.hs b/src/haskell/lib/Arion/Images.hs index cf52e22..369d09f 100644 --- a/src/haskell/lib/Arion/Images.hs +++ b/src/haskell/lib/Arion/Images.hs @@ -27,6 +27,7 @@ data Image = Image type TaggedImage = Text +-- | Subject to change loadImages :: FilePath -> IO () loadImages fp = do diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index 6eb40d9..bb00524 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -118,7 +118,6 @@ buildComposition outLink ea = do errDone <- async (BL.hPutStr stderr err) -- Force 'out' - -- TODO: use it? _v <- Protolude.evaluate out -- Wait for process exit and 'err' printout diff --git a/src/haskell/lib/Arion/Services.hs b/src/haskell/lib/Arion/Services.hs new file mode 100644 index 0000000..f63e6e2 --- /dev/null +++ b/src/haskell/lib/Arion/Services.hs @@ -0,0 +1,27 @@ +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DeriveAnyClass #-} +{-# LANGUAGE OverloadedStrings #-} +module Arion.Services + ( getDefaultExec + ) where + +import Prelude() +import Protolude hiding (to) + +import qualified Data.Aeson as Aeson +import Arion.Aeson (decodeFile) +import qualified Data.ByteString as BS +import qualified System.Process as Process + +import Control.Lens +import Data.Aeson.Lens +import Data.String +import System.IO (withFile, IOMode(ReadMode)) + +-- | Subject to change +getDefaultExec :: FilePath -> Text -> IO [Text] +getDefaultExec fp service = do + + v <- decodeFile fp + + pure ((v :: Aeson.Value) ^.. key "x-arion" . key "serviceInfo" . key service . key "defaultExec" . _Array . traverse . _String) From 3918799b9aef4449b2d610d0563020330100c8d8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 28 Sep 2019 14:53:46 +0200 Subject: [PATCH 22/27] Fix live-* script cabal file references --- live-check | 3 +-- live-unit-tests | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/live-check b/live-check index d7cbfb2..fb642a5 100755 --- a/live-check +++ b/live-check @@ -8,6 +8,5 @@ cd "$(dirname "${BASH_SOURCE[0]}")" ghcid \ --command 'ghci -isrc/haskell/exe src/haskell/exe/Main.hs' \ --reload=src/haskell \ - --restart=hercules-ci-api.cabal \ - --restart=../stack.yaml \ + --restart=arion-compose.cabal \ ; diff --git a/live-unit-tests b/live-unit-tests index 4f70f27..88a7e20 100755 --- a/live-unit-tests +++ b/live-unit-tests @@ -9,6 +9,5 @@ ghcid \ --command 'cabal v2-repl arion-compose:arion-unit-tests --flags ghci --write-ghc-environment-files=never' \ --test=Main.main \ --reload=src/haskell \ - --restart=hercules-ci-api.cabal \ - --restart=../stack.yaml \ + --restart=arion-compose.cabal \ ; From adc2e34deb47900a7c5411043ba2186450fdbf59 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 28 Sep 2019 15:27:19 +0200 Subject: [PATCH 23/27] Remove the bash implementation --- arion-compose.cabal | 1 - arion.nix | 39 ------ src/arion | 317 -------------------------------------------- 3 files changed, 357 deletions(-) delete mode 100644 arion.nix delete mode 100755 src/arion diff --git a/arion-compose.cabal b/arion-compose.cabal index 77891a7..18e4fb3 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -19,7 +19,6 @@ data-files: nix/*.nix , nix/modules/composition/*.nix , nix/modules/nixos/*.nix , nix/modules/service/*.nix - , arion-image/Dockerfile -- all data is verbatim from some sources data-dir: src diff --git a/arion.nix b/arion.nix deleted file mode 100644 index 2b4f81a..0000000 --- a/arion.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ stdenv, lib -, coreutils, docker_compose, jq -}: -let - - # TODO: Replace by a new expression for the new Haskell main - arion = stdenv.mkDerivation { - name = "arion"; - src = ./src; - unpackPhase = ""; - buildPhase = ""; - installPhase = '' - mkdir -p $out/bin $out/share/arion - cp -a nix $out/share/arion/ - cp -a arion-image $out/share/arion/ - tar -czf $out/share/arion/arion-image/tarball.tar.gz -C arion-image/tarball . - substitute arion $out/bin/arion \ - --subst-var-by path ${lib.makeBinPath [jq coreutils docker_compose]} \ - --subst-var-by nix_dir $out/share/arion/nix \ - ; - chmod a+x $out/bin/arion - ''; - inherit passthru; - }; - - passthru = { - inherit eval build; - }; - - eval = import "${nix_dir}/eval-composition.nix"; - - build = args@{...}: - let composition = eval args; - in composition.config.build.dockerComposeYaml; - - nix_dir = "${arion.outPath}/share/arion/nix"; - -in - arion diff --git a/src/arion b/src/arion deleted file mode 100755 index f8aff15..0000000 --- a/src/arion +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env bash - -# Close off the python module search path -# -# Accepting directories from the environment into the search path -# tends to break things. Docker Compose does not have a plugin -# system as far as I can tell, so I don't expect this to break a -# feature, but rather to make the program more robustly self- -# contained. -unset PYTHONPATH - -set -euo pipefail - -export PATH="@path@:$PATH" - -nix_dir="@nix_dir@" -docker_compose_args=() -files=() -command="docker-compose" -pkgs_argument="./arion-pkgs.nix" - -debug() { - # echo "$@" - : -} - -while test $# != 0; do - case "$1" in - -f|--file) - shift - files+=("$1") - ;; - -f*) - files+=("${1/#-f/}") - ;; - --file=*) - files+=("${1/#--file=}") - ;; - --pkgs) - shift - pkgs_argument="$1" - ;; - -h|--help|help) - command="help" - shift - break - ;; - cat) - command="$1" - shift - break - ;; - repl) - command="$1" - shift - break - ;; - exec) - command="$1" - shift - break - ;; - docker-compose) - command="docker-compose" - shift - break - ;; - *) - break - ;; - esac - shift -done - -while test $# != 0; do - docker_compose_args+=("$1") - shift -done - -case "$command" in - help) - cat <&2 "Evaluating configuration..." - # read-write-mode is required for import from derivation - nix-instantiate \ - "$nix_dir/eval-composition.nix" \ - --eval \ - --read-write-mode \ - --json \ - --argstr uid "$UID" \ - --arg modules "$modules" \ - --arg pkgs "$pkgs_argument" \ - --show-trace \ - --attr 'config.build.dockerComposeYamlText' \ - | jq -r . >$docker_compose_yaml; -} - -do_build() { - echo 1>&2 "Building configuration..." - nix-build \ - "$nix_dir/eval-composition.nix" \ - --out-link $docker_compose_yaml \ - --argstr uid "$UID" \ - --arg modules "$modules" \ - --arg pkgs "$pkgs_argument" \ - --show-trace \ - --attr 'config.build.dockerComposeYaml' \ - >/dev/null ; - - echo 1>&2 "Ensuring required images are loaded..." - jq -r <"$docker_compose_yaml" \ - '.["x-arion"].images | map(" - " + .imageName + ":" + .imageTag) | join("\n")' - eval "$( - jq -r '.["docker-compose"]["x-arion"].images as $images - | .["existing-images"] as $loaded - | $images - | map( - if $loaded[.imageName + ":" + .imageTag] - then "" - else "docker load <" + .image + ";" end - ) - | join("\n") - ' < arguments - # so we improvise. We need a file in this directory - # to make sure that all paths are as expected :( - trap do_repl_cleanup EXIT; - - REPL_TMP=.tmp-repl-$$-$RANDOM - cat <"$REPL_TMP" < Date: Sat, 28 Sep 2019 16:27:07 +0200 Subject: [PATCH 24/27] Rewire nix --- default.nix | 8 +++---- nix/arion.nix | 42 +++++++++++++++++++++++++++++++++++ nix/ci.nix | 6 +++++ nix/haskell-arion-compose.nix | 14 ++++++++++++ nix/haskell-overlay.nix | 11 +-------- nix/overlay.nix | 35 ++--------------------------- 6 files changed, 69 insertions(+), 47 deletions(-) create mode 100644 nix/arion.nix create mode 100644 nix/ci.nix create mode 100644 nix/haskell-arion-compose.nix diff --git a/default.nix b/default.nix index 7f48895..7b66620 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ -args@{ pkgs ? import ./nix args, system ? null, ... }: - +{ pkgs ? import ./nix {} +, haskellPackages ? pkgs.haskellPackages +}: { - inherit (pkgs) arion tests; - doc = pkgs.recurseIntoAttrs (import ./doc { inherit pkgs; }); + arion = import ./nix/arion.nix { inherit pkgs haskellPackages; }; } diff --git a/nix/arion.nix b/nix/arion.nix new file mode 100644 index 0000000..4b76fc1 --- /dev/null +++ b/nix/arion.nix @@ -0,0 +1,42 @@ +{ pkgs ? import ./. {} +, lib ? pkgs.lib +, haskellPackages ? pkgs.haskellPackages +, arion-compose ? import ./haskell-arion-compose.nix { inherit pkgs haskellPackages; } +}: + +let + inherit (pkgs.haskell.lib) justStaticExecutables overrideCabal; + + srcDir = ../src; + eval = import (srcDir + "/nix/eval-composition.nix"); + build = args@{...}: + let composition = eval args; + in composition.config.build.dockerComposeYaml; + +in + justStaticExecutables (overrideCabal arion-compose (o: { + buildTools = o.buildTools ++ [pkgs.makeWrapper]; + passthru = o.passthru // { + inherit eval build; + }; + pname = "arion"; # Cover up the needlessly long Haskell package name + + # PYTHONPATH + # + # We close off the python module search path! + # + # Accepting directories from the environment into the search path + # tends to break things. Docker Compose does not have a plugin + # system as far as I can tell, so I don't expect this to break a + # feature, but rather to make the program more robustly self- + # contained. + + postInstall = ''${o.postInstall or ""} + mkdir -p $out/libexec + mv $out/bin/arion $out/libexec + makeWrapper $out/libexec/arion $out/bin/arion \ + --unset PYTHONPATH \ + --prefix PATH : ${lib.makeBinPath [ pkgs.docker-compose ]} \ + ; + ''; + })) diff --git a/nix/ci.nix b/nix/ci.nix new file mode 100644 index 0000000..91f1a0f --- /dev/null +++ b/nix/ci.nix @@ -0,0 +1,6 @@ +args@{ pkgs ? import ./default.nix args, system ? null, ... }: + +{ + inherit (pkgs) arion tests; + doc = pkgs.recurseIntoAttrs (import ../doc { inherit pkgs; }); +} diff --git a/nix/haskell-arion-compose.nix b/nix/haskell-arion-compose.nix new file mode 100644 index 0000000..239d476 --- /dev/null +++ b/nix/haskell-arion-compose.nix @@ -0,0 +1,14 @@ + +# NOTE: This file produces a haskell library, not the arion package! + +{ pkgs ? import ./default.nix {}, haskellPackages ? pkgs.haskellPackages }: +let + inherit (pkgs.haskell.lib) overrideCabal addBuildTools; +in + overrideCabal (addBuildTools (haskellPackages.callCabal2nix "arion-compose" ./.. {}) [pkgs.nix]) (o: o // { + preCheck = '' + export NIX_LOG_DIR=$TMPDIR + export NIX_STATE_DIR=$TMPDIR + export NIX_PATH=nixpkgs=${pkgs.path} + ''; + }) \ No newline at end of file diff --git a/nix/haskell-overlay.nix b/nix/haskell-overlay.nix index 74e43c4..c2d82cb 100644 --- a/nix/haskell-overlay.nix +++ b/nix/haskell-overlay.nix @@ -1,13 +1,4 @@ self: super: hself: hsuper: -let - inherit (self.haskell.lib) addBuildTools overrideCabal; -in { - arion-compose = overrideCabal (addBuildTools (hself.callCabal2nix "arion-compose" ./.. {}) [self.nix]) (o: o // { - preCheck = '' - export NIX_LOG_DIR=$TMPDIR - export NIX_STATE_DIR=$TMPDIR - export NIX_PATH=nixpkgs=${self.path} - ''; - }); + arion-compose = import ./haskell-arion-compose.nix { pkgs = self; haskellPackages = hself; }; } \ No newline at end of file diff --git a/nix/overlay.nix b/nix/overlay.nix index 6a41e14..8b5490c 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -3,42 +3,10 @@ let inherit (self.arion-project) haskellPkgs; inherit (super) lib; - srcDir = ../src; # TODO gitignoreSource + whitelist nix and arion-image - eval = import (srcDir + "/nix/eval-composition.nix"); - build = args@{...}: - let composition = eval args; - in composition.config.build.dockerComposeYaml; - hlib = super.haskell.lib; in { - arion-v0 = super.callPackage ../arion.nix {}; - arion = hlib.justStaticExecutables (hlib.overrideCabal haskellPkgs.arion-compose (o: { - buildTools = o.buildTools ++ [super.makeWrapper]; - passthru = o.passthru // { - inherit eval build; - }; - pname = "arion"; # Cover up the needlessly long Haskell package name - - # PYTHONPATH - # - # We close off the python module search path! - # - # Accepting directories from the environment into the search path - # tends to break things. Docker Compose does not have a plugin - # system as far as I can tell, so I don't expect this to break a - # feature, but rather to make the program more robustly self- - # contained. - - postInstall = ''${o.postInstall or ""} - mkdir -p $out/libexec - mv $out/bin/arion $out/libexec - makeWrapper $out/libexec/arion $out/bin/arion \ - --unset PYTHONPATH \ - --prefix PATH : ${lib.makeBinPath [ self.docker-compose ]} \ - ; - ''; - })); + arion = import ./arion.nix { pkgs = self; }; tests = super.callPackage ../tests {}; doc = super.callPackage ../doc {}; @@ -50,6 +18,7 @@ in haskellPkgs.cabal-install haskellPkgs.ghcid super.docker-compose + (import ~/h/ghcide-nix {}).ghcide-ghc864 ]; }; }; From a90190fc9ea425b661a210fd48e1c205d3501297 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 29 Sep 2019 22:44:31 +0200 Subject: [PATCH 25/27] Fix stderr streaming Just good old-fashioned handles. --- arion-compose.cabal | 1 - src/haskell/lib/Arion/DockerCompose.hs | 2 - src/haskell/lib/Arion/Nix.hs | 58 ++++++++++++-------------- 3 files changed, 26 insertions(+), 35 deletions(-) diff --git a/arion-compose.cabal b/arion-compose.cabal index 18e4fb3..83aa6f6 100644 --- a/arion-compose.cabal +++ b/arion-compose.cabal @@ -33,7 +33,6 @@ common deps , lens , lens-aeson , process - , process-extras , temporary , text , protolude diff --git a/src/haskell/lib/Arion/DockerCompose.hs b/src/haskell/lib/Arion/DockerCompose.hs index 898eb9f..b7cce7f 100644 --- a/src/haskell/lib/Arion/DockerCompose.hs +++ b/src/haskell/lib/Arion/DockerCompose.hs @@ -9,8 +9,6 @@ import qualified Data.String import System.Process import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL -import qualified System.Process.ByteString.Lazy - as PBL import Paths_arion_compose import Control.Applicative diff --git a/src/haskell/lib/Arion/Nix.hs b/src/haskell/lib/Arion/Nix.hs index bb00524..cef6a92 100644 --- a/src/haskell/lib/Arion/Nix.hs +++ b/src/haskell/lib/Arion/Nix.hs @@ -18,8 +18,6 @@ import qualified System.Directory as Directory import System.Process import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as BL -import qualified System.Process.ByteString.Lazy - as PBL import Paths_arion_compose import Control.Applicative @@ -61,28 +59,30 @@ evaluateComposition ea = do ++ modeArguments (evalMode ea) ++ argArgs ea ++ map toS (evalUserArgs ea) - stdin = mempty - procSpec = (proc "nix-instantiate" args) { cwd = evalWorkDir ea } + procSpec = (proc "nix-instantiate" args) + { cwd = evalWorkDir ea + , std_out = CreatePipe + } - -- TODO: lazy IO is tricky. Let's use conduit/pipes instead? - (exitCode, out, err) <- PBL.readCreateProcessWithExitCode procSpec stdin + withCreateProcess procSpec $ \_in outHM _err procHandle -> do + let outHandle = fromMaybe (panic "stdout missing") outHM - -- Stream 'err' - errDone <- async (BL.hPutStr stderr err) + out <- BL.hGetContents outHandle - -- Force 'out' - v <- Protolude.evaluate (eitherDecode out) + v <- Protolude.evaluate (eitherDecode out) - -- Wait for process exit and 'err' printout - wait errDone + exitCode <- waitForProcess procHandle - case exitCode of - ExitSuccess -> pass - ExitFailure e -> throwIO $ FatalError "Evaluation failed" -- TODO: don't print this exception in main + case exitCode of + ExitSuccess -> pass + ExitFailure 1 -> exitFailure + e@ExitFailure {} -> do + throwIO $ FatalError $ "evaluation failed with " <> show exitCode + exitWith e - case v of - Right r -> pure r - Left e -> throwIO $ FatalError "Couldn't parse nix-instantiate output" + case v of + Right r -> pure r + Left e -> throwIO $ FatalError "Couldn't parse nix-instantiate output" -- | Run with docker-compose.yaml tmpfile withEvaluatedComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r @@ -108,24 +108,18 @@ buildComposition outLink ea = do ++ commandArgs ++ argArgs ea ++ map toS (evalUserArgs ea) - stdin = mempty procSpec = (proc "nix-build" args) { cwd = evalWorkDir ea } - -- TODO: lazy IO is tricky. Let's use conduit/pipes instead? - (exitCode, out, err) <- PBL.readCreateProcessWithExitCode procSpec stdin - - -- Stream 'err' - errDone <- async (BL.hPutStr stderr err) + withCreateProcess procSpec $ \_in _out _err procHandle -> do - -- Force 'out' - _v <- Protolude.evaluate out + exitCode <- waitForProcess procHandle - -- Wait for process exit and 'err' printout - wait errDone - - case exitCode of - ExitSuccess -> pass - ExitFailure e -> throwIO $ FatalError "Build failed" -- TODO: don't print this exception in main + case exitCode of + ExitSuccess -> pass + ExitFailure 1 -> exitFailure + e@ExitFailure {} -> do + throwIO $ FatalError $ "nix-build failed with " <> show exitCode + exitWith e -- | Do something with a docker-compose.yaml. withBuiltComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r From 02d319acf615252f080c08f0593eab41a7ef2c2d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 29 Sep 2019 23:44:53 +0200 Subject: [PATCH 26/27] Expose composition to services --- doc/manual/default.nix | 2 +- src/nix/eval-service.nix | 5 +++-- src/nix/modules/composition/docker-compose.nix | 6 +++++- src/nix/modules/service/{host.nix => context.nix} | 6 ++++++ 4 files changed, 15 insertions(+), 4 deletions(-) rename src/nix/modules/service/{host.nix => context.nix} (56%) diff --git a/doc/manual/default.nix b/doc/manual/default.nix index ff2096c..03522ab 100644 --- a/doc/manual/default.nix +++ b/doc/manual/default.nix @@ -69,7 +69,7 @@ let declarations = map (d: "src/nix" + (lib.strings.removePrefix (toString ${src}) (toString d))) opt.declarations; }; inherit (pkgs) lib; - composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; name = abort "The manual's service options section must not depend on the service name."; }; + composition = pkgs.callPackage ${src}/eval-service.nix {} { modules = []; host = {}; name = abort "The manual's service options section must not depend on the service name."; composition = abort "The manual's service options must not depend on the composition."; }; in map fixPaths (lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList composition.options)) ''; }; diff --git a/src/nix/eval-service.nix b/src/nix/eval-service.nix index cb866ba..2ab731b 100644 --- a/src/nix/eval-service.nix +++ b/src/nix/eval-service.nix @@ -1,6 +1,6 @@ { lib, pkgs, ... }: -{ modules, host, name }: +{ modules, host, name, composition }: let composite = lib.evalModules { check = true; @@ -13,7 +13,7 @@ let ./modules/service/docker-compose-service.nix ./modules/service/extended-info.nix ./modules/service/host-store.nix - ./modules/service/host.nix + ./modules/service/context.nix ./modules/service/image.nix ./modules/service/nixos.nix ./modules/service/nixos-init.nix @@ -25,6 +25,7 @@ let config._module.args.pkgs = lib.mkForce pkgs; config.host = host; config.service.name = name; + config.composition = composition; }; in diff --git a/src/nix/modules/composition/docker-compose.nix b/src/nix/modules/composition/docker-compose.nix index c223f68..1891bbf 100644 --- a/src/nix/modules/composition/docker-compose.nix +++ b/src/nix/modules/composition/docker-compose.nix @@ -11,7 +11,11 @@ */ { pkgs, lib, config, ... }: let - evalService = name: modules: pkgs.callPackage ../../eval-service.nix {} { inherit name modules; inherit (config) host; }; + evalService = name: modules: pkgs.callPackage ../../eval-service.nix {} { + inherit name modules; + inherit (config) host; + composition = config; + }; in { diff --git a/src/nix/modules/service/host.nix b/src/nix/modules/service/context.nix similarity index 56% rename from src/nix/modules/service/host.nix rename to src/nix/modules/service/context.nix index 3ba2ee8..10854e2 100644 --- a/src/nix/modules/service/host.nix +++ b/src/nix/modules/service/context.nix @@ -7,5 +7,11 @@ The composition-level host option values. ''; }; + composition = lib.mkOption { + type = lib.types.attrs; + description = '' + The composition configuration. + ''; + }; }; } From c88d2bb9cf0719140465e0b4158d672a1efdb497 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 29 Sep 2019 23:45:52 +0200 Subject: [PATCH 27/27] docker load arion-base-image --- .../testdata/Arion/NixSpec/arion-compose.json | 4 +- src/nix/eval-composition.nix | 1 + .../modules/composition/arion-base-image.nix | 41 +++++++++++++++++++ src/nix/modules/service/host-store.nix | 10 +---- 4 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 src/nix/modules/composition/arion-base-image.nix diff --git a/src/haskell/testdata/Arion/NixSpec/arion-compose.json b/src/haskell/testdata/Arion/NixSpec/arion-compose.json index 37a9051..4b389e1 100644 --- a/src/haskell/testdata/Arion/NixSpec/arion-compose.json +++ b/src/haskell/testdata/Arion/NixSpec/arion-compose.json @@ -9,7 +9,7 @@ "PATH": "/usr/bin:/run/current-system/sw/bin/", "container": "docker" }, - "image": "webserver:", + "image": "arion-base:", "ports": [ "8000:80" ], @@ -33,7 +33,7 @@ "images": [ { "image": "", - "imageName": "webserver", + "imageName": "arion-base", "imageTag": "" } ], diff --git a/src/nix/eval-composition.nix b/src/nix/eval-composition.nix index b3fc78a..d4a2b57 100644 --- a/src/nix/eval-composition.nix +++ b/src/nix/eval-composition.nix @@ -22,6 +22,7 @@ let ./modules/composition/host-environment.nix ./modules/composition/images.nix ./modules/composition/service-info.nix + ./modules/composition/arion-base-image.nix ]; argsModule = { diff --git a/src/nix/modules/composition/arion-base-image.nix b/src/nix/modules/composition/arion-base-image.nix new file mode 100644 index 0000000..fad7b6c --- /dev/null +++ b/src/nix/modules/composition/arion-base-image.nix @@ -0,0 +1,41 @@ + + +# This module is subject to change. +# In particular, arion-base should use a generic non-service image building system + +{ config, lib, pkgs, ... }: + +let + + tag = lib.head (lib.strings.splitString "-" (baseNameOf builtImage.outPath)); + name = "arion-base"; + + builtImage = pkgs.dockerTools.buildLayeredImage { + inherit name; + contents = pkgs.runCommand "minimal-contents" {} '' + mkdir -p $out/bin $out/usr/bin + ln -s /run/system/bin/sh $out/bin/sh + ln -s /run/system/usr/bin/env $out/usr/bin/env + ''; + config = {}; + }; + +in + +{ + + options = { + arionBaseImage = lib.mkOption { + type = lib.types.str; + description = "Image to use when using useHostStore. Don't use this option yourself. It's going away."; + internal = true; + }; + }; + + config = { + arionBaseImage = "${name}:${tag}"; + build.imagesToLoad = lib.mkIf (lib.any (s: s.config.service.useHostStore) (lib.attrValues config.docker-compose.evaluatedServices)) [ + { image = builtImage; imageName = name; imageTag = tag; } + ]; + }; +} \ No newline at end of file diff --git a/src/nix/modules/service/host-store.nix b/src/nix/modules/service/host-store.nix index f3d1c08..0c9f1cb 100644 --- a/src/nix/modules/service/host-store.nix +++ b/src/nix/modules/service/host-store.nix @@ -29,14 +29,8 @@ in }; }; config = mkIf config.service.useHostStore { - image.nixBuild = true; - image.contents = [ - (pkgs.runCommand "minimal-contents" {} '' - mkdir -p $out/bin $out/usr/bin - ln -s /run/system/bin/sh $out/bin/sh - ln -s /run/system/usr/bin/env $out/usr/bin/env - '') - ]; + image.nixBuild = false; # no need to build and load + service.image = config.composition.arionBaseImage; service.environment.NIX_REMOTE = lib.optionalString config.service.useHostNixDaemon "daemon"; service.volumes = [ "${config.host.nixStorePrefix}/nix/store:/nix/store${lib.optionalString config.service.hostStoreAsReadOnly ":ro"}"