From 29599f529b9a1647bce623d72dd3e22ee0c743cc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Oct 2020 16:22:15 +0200 Subject: [PATCH 1/7] Add basic flake --- flake.lock | 25 +++++++++++++++++++++++++ flake.nix | 27 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..4e92b89 --- /dev/null +++ b/flake.lock @@ -0,0 +1,25 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1601906239, + "narHash": "sha256-P1jBYbYeFswig/0FKbgh+BpVhh9iurD3m0T2ae4gdx8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c2bb4af48d26ed091e5674394bacbf8d488c7939", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..fe4d516 --- /dev/null +++ b/flake.nix @@ -0,0 +1,27 @@ +{ + description = "Arion - use Docker Compose via Nix"; + + outputs = { self, nixpkgs }: + let + lib = import (nixpkgs + "/lib"); + systems = [ + "aarch64-linux" + "x86_64-darwin" + "x86_64-linux" + ]; + in { + + packages = lib.genAttrs systems (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + arion = import ./nix/arion.nix { inherit pkgs; }; + }); + + defaultPackage = lib.genAttrs systems (system: + self.packages.${system}.arion + ); + + }; +} From 7340d37636ae59ef8de8a07b5af78dfa52f7fdf5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Oct 2020 17:01:25 +0200 Subject: [PATCH 2/7] Use nixpkgs upstream expression --- nix/arion.nix | 44 +++--------------- nix/upstreamable/default.nix | 89 ++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 nix/upstreamable/default.nix diff --git a/nix/arion.nix b/nix/arion.nix index b53b1f4..0374853 100644 --- a/nix/arion.nix +++ b/nix/arion.nix @@ -1,42 +1,12 @@ +# Like the upstreamable expression but wired up for the local arion. { pkgs ? import ./. {} , lib ? pkgs.lib +, haskell ? pkgs.haskell , haskellPackages ? pkgs.haskellPackages , arion-compose ? import ./haskell-arion-compose.nix { inherit pkgs haskellPackages; } +, runCommand ? pkgs.runCommand }: - -let - inherit (pkgs.haskell.lib) justStaticExecutables overrideCabal; - - srcDir = ../src; - eval = import (srcDir + "/nix/eval-composition.nix"); - build = args@{...}: - let composition = eval args; - in composition.config.out.dockerComposeYaml; - -in - justStaticExecutables (overrideCabal arion-compose (o: { - buildTools = o.buildTools ++ [pkgs.makeWrapper]; - passthru = o.passthru // { - inherit eval build; - }; - pname = "arion"; # Cover up the needlessly long Haskell package name - - # PYTHONPATH - # - # We close off the python module search path! - # - # Accepting directories from the environment into the search path - # tends to break things. Docker Compose does not have a plugin - # system as far as I can tell, so I don't expect this to break a - # feature, but rather to make the program more robustly self- - # contained. - - postInstall = ''${o.postInstall or ""} - mkdir -p $out/libexec - mv $out/bin/arion $out/libexec - makeWrapper $out/libexec/arion $out/bin/arion \ - --unset PYTHONPATH \ - --prefix PATH : ${lib.makeBinPath [ pkgs.docker-compose ]} \ - ; - ''; - })) +import ./upstreamable/default.nix { + inherit pkgs lib haskell runCommand; + haskellPackages = haskellPackages // { inherit arion-compose; }; +} diff --git a/nix/upstreamable/default.nix b/nix/upstreamable/default.nix new file mode 100644 index 0000000..149beec --- /dev/null +++ b/nix/upstreamable/default.nix @@ -0,0 +1,89 @@ +{ pkgs +, lib +, haskellPackages +, haskell +, runCommand +}: + +let + + /* This derivation builds the arion tool. + + It is based on the arion-compose Haskell package, but adapted and extended to + - have the correct name + - have a smaller closure size + - have functions to use Arion from inside Nix: arion.eval and arion.build + - make it self-contained by including docker-compose + */ + arion = + justStaticExecutables ( + overrideCabal + arion-compose + cabalOverrides + ); + + inherit (haskell.lib) justStaticExecutables overrideCabal; + + inherit (haskellPackages) arion-compose; + + cabalOverrides = o: { + buildTools = (o.buildTools or []) ++ [pkgs.makeWrapper]; + passthru = (o.passthru or {}) // { + inherit eval build; + }; + # Patch away the arion-compose name. Unlike the Haskell library, the program + # is called arion (arion was already taken on hackage). + pname = "arion"; + src = arion-compose.src; + + # PYTHONPATH + # + # We close off the python module search path! + # + # Accepting directories from the environment into the search path + # tends to break things. Docker Compose does not have a plugin + # system as far as I can tell, so I don't expect this to break a + # feature, but rather to make the program more robustly self- + # contained. + + postInstall = ''${o.postInstall or ""} + mkdir -p $out/libexec + mv $out/bin/arion $out/libexec + makeWrapper $out/libexec/arion $out/bin/arion \ + --unset PYTHONPATH \ + --prefix PATH : ${lib.makeBinPath [ pkgs.docker-compose ]} \ + ; + ''; + }; + + # Unpacked sources for evaluation by `eval` + srcUnpacked = pkgs.stdenv.mkDerivation { + name = "arion-src"; + inherit (arion-compose) src; + buildPhase = '' + cp -r $src $out + ''; + installPhase = ""; + }; + + /* Function for evaluating a composition + + Re-uses this Nixpkgs evaluation instead of `arion-pkgs.nix`. + + Returns the module system's `config` and `options` variables. + */ + eval = args@{...}: + import (srcUnpacked + "/src/nix/eval-composition.nix") + ({ inherit pkgs; } // args); + + /* Function to derivation of the docker compose yaml file + NOTE: The output will change: https://github.com/hercules-ci/arion/issues/82 + + This function is particularly useful on CI, although the references + to image tarballs may not always be desirable. + */ + build = args@{...}: + let composition = eval args; + in composition.config.out.dockerComposeYaml; + +in arion From 8cb231fa899dfd75306d0d829a42abdac797bba2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Oct 2020 17:04:44 +0200 Subject: [PATCH 3/7] Update comment about tarballs --- nix/upstreamable/default.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nix/upstreamable/default.nix b/nix/upstreamable/default.nix index 149beec..8f874ec 100644 --- a/nix/upstreamable/default.nix +++ b/nix/upstreamable/default.nix @@ -79,8 +79,9 @@ let /* Function to derivation of the docker compose yaml file NOTE: The output will change: https://github.com/hercules-ci/arion/issues/82 - This function is particularly useful on CI, although the references - to image tarballs may not always be desirable. + This function is particularly useful on CI. On Nixpkgs >= 20.09 this will + not store the image tarballs but executables to produce them reliably via + streamLayeredImage. */ build = args@{...}: let composition = eval args; From 39249c5956821e988f7c8782daa6f55a5e8f5e1a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Oct 2020 17:36:39 +0200 Subject: [PATCH 4/7] flake: Add build and eval functions --- flake.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flake.nix b/flake.nix index fe4d516..8388e0b 100644 --- a/flake.nix +++ b/flake.nix @@ -19,9 +19,17 @@ arion = import ./nix/arion.nix { inherit pkgs; }; }); + # Does not include the eval and build functions like you may expect from Nixpkgs. defaultPackage = lib.genAttrs systems (system: self.packages.${system}.arion ); + lib = { + eval = import ./src/nix/eval-composition.nix; + build = args@{...}: + let composition = self.lib.eval args; + in composition.config.out.dockerComposeYaml; + }; + }; } From 525b598ce37688ed7c19c4bfd82850688d41a1d8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 5 Oct 2020 17:38:48 +0200 Subject: [PATCH 5/7] flake: Add overlay --- flake.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8388e0b..d711a13 100644 --- a/flake.nix +++ b/flake.nix @@ -9,14 +9,21 @@ "x86_64-darwin" "x86_64-linux" ]; + arionFromPkgs = pkgs: import ./nix/arion.nix { inherit pkgs; }; in { + # The overlay is currently the recommended way to integrate arion, + # because its arion attribute behaves just like Nixpkgs. + overlay = final: prev: { + arion = arionFromPkgs final; + }; + packages = lib.genAttrs systems (system: let pkgs = nixpkgs.legacyPackages.${system}; in { - arion = import ./nix/arion.nix { inherit pkgs; }; + arion = arionFromPkgs pkgs; }); # Does not include the eval and build functions like you may expect from Nixpkgs. From 3fb8782296eb04761811defc4beaff9c910a1300 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 10 Oct 2020 21:27:48 +0200 Subject: [PATCH 6/7] Allow overriding the source used for evaluation --- nix/arion.nix | 1 + nix/upstreamable/default.nix | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/nix/arion.nix b/nix/arion.nix index 0374853..e43fd9e 100644 --- a/nix/arion.nix +++ b/nix/arion.nix @@ -9,4 +9,5 @@ import ./upstreamable/default.nix { inherit pkgs lib haskell runCommand; haskellPackages = haskellPackages // { inherit arion-compose; }; + evalSrc = ./..; } diff --git a/nix/upstreamable/default.nix b/nix/upstreamable/default.nix index 8f874ec..a06b42d 100644 --- a/nix/upstreamable/default.nix +++ b/nix/upstreamable/default.nix @@ -1,8 +1,14 @@ +args@ { pkgs , lib , haskellPackages , haskell , runCommand + + # Allow this expression file to be used more efficiently in situations where + # the sources are more readily available. Unpacking haskellPackages.arion-compose.src + # is not always the best choice for arion.eval. +, evalSrc ? null }: let @@ -57,14 +63,8 @@ let }; # Unpacked sources for evaluation by `eval` - srcUnpacked = pkgs.stdenv.mkDerivation { - name = "arion-src"; - inherit (arion-compose) src; - buildPhase = '' - cp -r $src $out - ''; - installPhase = ""; - }; + evalSrc' = args.evalSrc or (runCommand "arion-src" {} + "mkdir $out; tar -C $out --strip-components=1 -xf ${arion-compose.src}"); /* Function for evaluating a composition @@ -73,7 +73,7 @@ let Returns the module system's `config` and `options` variables. */ eval = args@{...}: - import (srcUnpacked + "/src/nix/eval-composition.nix") + import (evalSrc' + "/src/nix/eval-composition.nix") ({ inherit pkgs; } // args); /* Function to derivation of the docker compose yaml file From e1f7840780ed21a05064d0416a3810614dda55e8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 28 Oct 2020 11:43:03 +0100 Subject: [PATCH 7/7] Update doc about garbage collection --- docs/modules/ROOT/pages/index.adoc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index 5f900e5..37eb06f 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -244,13 +244,12 @@ Nope, it’s just Nix and Docker Compose under the hood. === What about garbage collection? Arion removes the need for garbage collecting docker images, delegating -this task to Nix. +this task to Nix when using `service.useHostStore`. -Arion creates a garbage collection root and cleans it up after -completing the command. This means that `arion up` without `-d` is safe -with respect to garbage collection. A deployment that is more serious -than local development must leave a GC root on the deployment host. This -use case is not supported as of now. +Arion creates a garbage collection root that it cleans up after completing +the command. This means that `arion up -d` should not be used with `useHostStore` +in production. Instead, disable `useHostStore`, which will use `dockerTools` to +generate images that can be used in production. === Why is my container not running latest code?