Merge pull request #74 from hercules-ci/release-prep

Release prep, release
This commit is contained in:
Robert Hensing 2019-10-07 11:27:59 +02:00 committed by GitHub
commit 9fedd0e152
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 277 additions and 93 deletions

View file

@ -1,6 +1,9 @@
# Revision history for arion-compose
# Revision history for Arion
## 0.1.0.0 -- YYYY-mm-dd
## Unreleased
* First version. Released on an unsuspecting world.
* *BREAKING:* useHostStore now uses a proper empty base image, like `scratch`.
<!-- TODO: use better template -->
## 0.1.0.0 -- 2019-10-04
* First released version. Released on an unsuspecting world.

View file

@ -3,7 +3,7 @@ cabal-version: 2.4
name: arion-compose
version: 0.1.0.0
synopsis: Run docker-compose with help from Nix/NixOS
-- description:
description: Arion is a tool for building and running applications that consist of multiple docker containers using NixOS modules. It has special support for docker images that are built with Nix, for a smooth development experience and improved performance.
homepage: https://github.com/hercules-ci/arion#readme
-- bug-reports:
license: Apache-2.0
@ -11,10 +11,10 @@ license-file: LICENSE
author: Robert Hensing
maintainer: robert@hercules-ci.com
-- copyright:
-- category:
extra-source-files: CHANGELOG.md, README.asciidoc
write-ghc-enviroment-files:
never
category: Distribution, Network, Cloud, Distributed Computing
extra-source-files: CHANGELOG.md, README.asciidoc,
src/haskell/testdata/**/*.nix
src/haskell/testdata/**/*.json
data-files: nix/*.nix
, nix/modules/composition/*.nix
, nix/modules/nixos/*.nix
@ -24,7 +24,7 @@ data-files: nix/*.nix
-- all data is verbatim from some sources
data-dir: src
common deps
common common
build-depends: base ^>=4.12.0.0
, aeson
, aeson-pretty
@ -38,25 +38,27 @@ common deps
, text
, protolude
, unix
ghc-options: -Wall
flag ghci
default: False
manual: True
library
import: deps
import: common
exposed-modules: Arion.Nix
Arion.Aeson
Arion.DockerCompose
Arion.Images
Arion.Services
other-modules: Paths_arion_compose
autogen-modules: Paths_arion_compose
-- other-extensions:
hs-source-dirs: src/haskell/lib
default-language: Haskell2010
executable arion
import: deps
import: common
main-is: Main.hs
-- other-modules:
-- other-extensions:
@ -66,7 +68,7 @@ executable arion
default-language: Haskell2010
test-suite arion-unit-tests
import: deps
import: common
if flag(ghci)
hs-source-dirs: src/haskell/lib
ghc-options: -Wno-missing-home-modules

View file

@ -1,6 +1,47 @@
args@{ pkgs ? import ./default.nix args, system ? null, ... }:
let
sources = import ./sources.nix;
lib = import (sources."nixpkgs" + "/lib");
inherit (import sources."project.nix" { inherit lib; }) dimension;
in
dimension "Nixpkgs version" {
"nixos-19_03" = {
nixpkgsSource = "nixpkgs";
isReferenceNixpkgs = true;
};
"nixos-19_09" = {
nixpkgsSource = "nixos-19.09";
# Broken since 19.09, wontfix because doc tooling will be changed.
# TODO: reenable
enableDoc = false;
};
"nixos-unstable" = {
nixpkgsSource = "nixos-unstable";
# Broken since 19.09, wontfix because doc tooling will be changed.
# TODO: reenable
enableDoc = false;
};
} (
_name: { nixpkgsSource, isReferenceNixpkgs ? false, enableDoc ? true }:
dimension "System" {
"x86_64-linux" = { isReferenceTarget = isReferenceNixpkgs; };
# TODO: darwin
# "x86_64-darwin" = { enableNixOSTests = false; };
} (
system: { isReferenceTarget ? false }:
let
pkgs = import ./. { inherit system; nixpkgsSrc = sources.${nixpkgsSource}; };
in
{
inherit (pkgs) arion tests;
} // lib.optionalAttrs enableDoc {
doc = pkgs.recurseIntoAttrs (import ../doc { inherit pkgs; });
} // lib.optionalAttrs isReferenceTarget {
inherit (pkgs.arion-project.haskellPkgs) arion-compose-checked;
}
)
)

View file

@ -1,12 +1,10 @@
/**
* This is the entry-point for all nix execution in this project.
*/
{ nixpkgsSrc ? ./nixpkgs.nix
, system ? null
{ sources ? import ./sources.nix
, nixpkgsSrc ? sources.nixpkgs
, system ? builtins.currentSystem
, ...
}:
import (import ./nixpkgs.nix) ({
import nixpkgsSrc ({
# Makes the config pure as well. See <nixpkgs>/top-level/impure.nix:
config = {
};
@ -14,6 +12,5 @@ import (import ./nixpkgs.nix) ({
# all the packages are defined there:
(import ./overlay.nix)
];
} // (if system == null then {} else {
inherit system;
}))
})

View file

@ -1,4 +1,16 @@
self: super: hself: hsuper:
{
arion-compose = import ./haskell-arion-compose.nix { pkgs = self; haskellPackages = hself; };
arion-compose-checked =
let pkg = super.haskell.lib.buildStrictly hself.arion-compose;
checked = super.haskell.lib.overrideCabal pkg (o: {
postConfigure = ''${o.postConfigure or ""}
if ! ${hsuper.cabal-install}/bin/cabal check;
then
echo 1>&2 ERROR: cabal file is invalid. Above warnings were errors.
exit 1
fi
'';
});
in checked;
}

View file

@ -1,5 +0,0 @@
# to update: $ nix-prefetch-url --unpack url
builtins.fetchTarball {
url = "https://github.com/NixOS/nixpkgs/archive/bd5e8f35c2e9d1ddc9cd2fea7a23563336d54acb.tar.gz";
sha256 = "1wnzqqijrwf797nb234q10zb1h7086njradkkrx3a15b303grsw4";
}

View file

@ -3,10 +3,12 @@ let
inherit (self.arion-project) haskellPkgs;
inherit (super) lib;
sources = import ./sources.nix;
in
{
arion = import ./arion.nix { pkgs = self; };
inherit (import ./.. { pkgs = self; }) arion;
tests = super.callPackage ../tests {};
doc = super.callPackage ../doc {};
@ -19,7 +21,12 @@ in
haskellPkgs.ghcid
super.docker-compose
(import ~/h/ghcide-nix {}).ghcide-ghc864
self.niv
self.releaser
];
};
};
inherit (import (sources.niv) {}) niv;
releaser = self.haskellPackages.callCabal2nix "releaser" sources.releaser {};
}

75
nix/sources.json Normal file
View file

@ -0,0 +1,75 @@
{
"niv": {
"branch": "master",
"description": "Easy dependency management for Nix projects",
"homepage": "https://github.com/nmattia/niv",
"owner": "nmattia",
"repo": "niv",
"rev": "1dd094156b249586b66c16200ecfd365c7428dc0",
"sha256": "1b2vjnn8iac5iiqszjc2v1s1ygh0yri998c0k3s4x4kn0dsqik21",
"type": "tarball",
"url": "https://github.com/nmattia/niv/archive/1dd094156b249586b66c16200ecfd365c7428dc0.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixos-19.09": {
"branch": "nixos-19.09",
"description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
"homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS",
"repo": "nixpkgs-channels",
"rev": "3ba0d9f75ccffd41e32cfea4046805f8bbab12f5",
"sha256": "0w20drs4mwlq12k1sss1x8adyf5ph5jd52n8wdcgmn4sm60qjmki",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/3ba0d9f75ccffd41e32cfea4046805f8bbab12f5.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixos-unstable": {
"branch": "nixos-unstable",
"description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
"homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS",
"repo": "nixpkgs-channels",
"rev": "cb4332e3eb6dfdb653f1fc7397a0292df228a533",
"sha256": "1722wphznqhpfny08rcy19l85r2l893ckjc3h1vfivj6aj64fwjr",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/cb4332e3eb6dfdb653f1fc7397a0292df228a533.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
"version": ""
},
"nixpkgs": {
"branch": "nixos-19.03",
"description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
"homepage": "https://github.com/NixOS/nixpkgs",
"owner": "NixOS",
"repo": "nixpkgs-channels",
"rev": "6420e2649fa9e267481fb78e602022dab9d1dcd1",
"sha256": "1z3hx7gp8nxk3fspi8vik3j9zxpajj3s7nxvjvx3b5igndxwbp74",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs-channels/archive/6420e2649fa9e267481fb78e602022dab9d1dcd1.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"project.nix": {
"branch": "master",
"description": "A configuration manager for your projects",
"homepage": null,
"owner": "hercules-ci",
"repo": "project.nix",
"rev": "33e5f3cb25feff4ccd00f8c60a05976e2ee01802",
"sha256": "0c3q3il5h6q3ms8m6da51knvjsfvpz12sh3a3av4d2a5ikm5ncl1",
"type": "tarball",
"url": "https://github.com/hercules-ci/project.nix/archive/33e5f3cb25feff4ccd00f8c60a05976e2ee01802.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"releaser": {
"branch": "master",
"description": "Automation of Haskell package release process.",
"homepage": null,
"owner": "domenkozar",
"repo": "releaser",
"rev": "43a4e27654f388e8eacab631e24e26792ff88fe2",
"sha256": "072jlbw0hdc4nvs9frd7wdyzdv4mz2dc5ib35iaqi9rzdafq6822",
"type": "tarball",
"url": "https://github.com/domenkozar/releaser/archive/43a4e27654f388e8eacab631e24e26792ff88fe2.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}

93
nix/sources.nix Normal file
View file

@ -0,0 +1,93 @@
# This file has been generated by Niv.
# A record, from name to path, of the third-party packages
with rec
{
pkgs =
if hasNixpkgsPath
then
if hasThisAsNixpkgsPath
then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}
else import <nixpkgs> {}
else
import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {};
sources_nixpkgs =
if builtins.hasAttr "nixpkgs" sources
then sources.nixpkgs
else abort
''
Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
add a package called "nixpkgs" to your sources.json.
'';
# fetchTarball version that is compatible between all the versions of Nix
builtins_fetchTarball =
{ url, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchTarball;
in
if lessThan nixVersion "1.12" then
fetchTarball { inherit url; }
else
fetchTarball attrs;
# fetchurl version that is compatible between all the versions of Nix
builtins_fetchurl =
{ url, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchurl;
in
if lessThan nixVersion "1.12" then
fetchurl { inherit url; }
else
fetchurl attrs;
# A wrapper around pkgs.fetchzip that has inspectable arguments,
# annoyingly this means we have to specify them
fetchzip = { url, sha256 }@attrs: pkgs.fetchzip attrs;
# A wrapper around pkgs.fetchurl that has inspectable arguments,
# annoyingly this means we have to specify them
fetchurl = { url, sha256 }@attrs: pkgs.fetchurl attrs;
hasNixpkgsPath = (builtins.tryEval <nixpkgs>).success;
hasThisAsNixpkgsPath =
(builtins.tryEval <nixpkgs>).success && <nixpkgs> == ./.;
sources = builtins.fromJSON (builtins.readFile ./sources.json);
mapAttrs = builtins.mapAttrs or
(f: set: with builtins;
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)));
# borrowed from nixpkgs
functionArgs = f: f.__functionArgs or (builtins.functionArgs f);
callFunctionWith = autoArgs: f: args:
let auto = builtins.intersectAttrs (functionArgs f) autoArgs;
in f (auto // args);
getFetcher = spec:
let fetcherName =
if builtins.hasAttr "type" spec
then builtins.getAttr "type" spec
else "builtin-tarball";
in builtins.getAttr fetcherName {
"tarball" = fetchzip;
"builtin-tarball" = builtins_fetchTarball;
"file" = fetchurl;
"builtin-url" = builtins_fetchurl;
};
};
# NOTE: spec must _not_ have an "outPath" attribute
mapAttrs (_: spec:
if builtins.hasAttr "outPath" spec
then abort
"The values in sources.json should not have an 'outPath' attribute"
else
if builtins.hasAttr "url" spec && builtins.hasAttr "sha256" spec
then
spec //
{ outPath = callFunctionWith spec (getFetcher spec) { }; }
else spec
) sources

View file

@ -12,20 +12,13 @@ import qualified Arion.DockerCompose as DockerCompose
import Arion.Services (getDefaultExec)
import Options.Applicative
import Control.Applicative
import Control.Monad.Fail
import qualified Data.Aeson.Encode.Pretty
import qualified Data.Text as T
import qualified Data.Text.IO as T
import qualified Data.Text.Lazy as TL
import qualified Data.Text.Lazy.Builder as TB
import qualified Data.List.NonEmpty as NE
import Data.List.NonEmpty (NonEmpty(..))
import Control.Arrow ((>>>))
import System.Posix.User (getRealUserID)
data CommonOptions =
@ -69,6 +62,7 @@ parseOptions = do
let nixArgs = userNixArgs <|> "--show-trace" <$ guard showTrace
in CommonOptions{..}
textArgument :: Mod ArgumentFields [Char] -> Parser Text
textArgument = fmap T.pack . strArgument
parseCommand :: Parser (CommonOptions -> IO ())
@ -124,18 +118,18 @@ commandDC
-> Text
-> Text
-> Mod CommandFields (CommonOptions -> IO ())
commandDC run cmdStr help =
commandDC run cmdStr helpText =
command
(T.unpack cmdStr)
(info
(run cmdStr <$> parseDockerComposeArgs)
(progDesc (T.unpack help) <> fullDesc <> forwardOptions))
(progDesc (T.unpack helpText) <> fullDesc <> forwardOptions))
--------------------------------------------------------------------------------
runDC :: Text -> DockerComposeArgs -> CommonOptions -> IO ()
runDC cmd (DockerComposeArgs args) opts = do
runDC cmd (DockerComposeArgs args) _opts = do
DockerCompose.run DockerCompose.Args
{ files = []
, otherArgs = [cmd] ++ args
@ -265,7 +259,7 @@ runExec detach privileged user noTTY index envs workDir service commandAndArgs o
main :: IO ()
main =
(join . execParser) (info (parseAll <**> helper) fullDesc)
(join . arionExecParser) (info (parseAll <**> helper) fullDesc)
where
execParser = customExecParser (prefs showHelpOnEmpty)
arionExecParser = customExecParser (prefs showHelpOnEmpty)

View file

@ -4,11 +4,9 @@ import Prelude ()
import Data.Aeson
import qualified Data.ByteString.Lazy as BL
import qualified Data.Text.Lazy as TL
import qualified Data.Text.Lazy.IO as TL
import qualified Data.Text.Lazy.Builder as TB
import qualified Data.Aeson.Encode.Pretty
import Data.Aeson.Encode.Pretty ( defConfig
, keyOrder
, confCompare
, confTrailingNewline
)

View file

@ -3,24 +3,7 @@ module Arion.DockerCompose where
import Prelude ( )
import Protolude
import Arion.Aeson ( pretty )
import Data.Aeson
import qualified Data.String
import System.Process
import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as BL
import Paths_arion_compose
import Control.Applicative
import qualified Data.Text as T
import qualified Data.Text.IO as T
import qualified Data.List.NonEmpty as NE
import Data.List.NonEmpty ( NonEmpty(..) )
import Control.Arrow ( (>>>) )
import System.IO.Temp ( withTempFile )
import System.IO ( hClose )
data Args = Args
{ files :: [FilePath]
@ -43,6 +26,5 @@ run args = do
case exitCode of
ExitSuccess -> pass
ExitFailure 1 -> exitFailure
e@ExitFailure {} -> do
ExitFailure {} -> do
throwIO $ FatalError $ "docker-compose failed with " <> show exitCode
exitWith e

View file

@ -10,7 +10,6 @@ import Protolude hiding (to)
import qualified Data.Aeson as Aeson
import Arion.Aeson (decodeFile)
import qualified Data.ByteString as BS
import qualified System.Process as Process
import Control.Lens

View file

@ -16,12 +16,9 @@ import Data.Aeson
import qualified Data.String
import qualified System.Directory as Directory
import System.Process
import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as BL
import Paths_arion_compose
import Control.Applicative
import qualified Data.Text as T
import qualified Data.Text.IO as T
import qualified Data.List.NonEmpty as NE
@ -76,21 +73,20 @@ evaluateComposition ea = do
case exitCode of
ExitSuccess -> pass
ExitFailure 1 -> exitFailure
e@ExitFailure {} -> do
ExitFailure {} -> do
throwIO $ FatalError $ "evaluation failed with " <> show exitCode
exitWith e
case v of
Right r -> pure r
Left e -> throwIO $ FatalError "Couldn't parse nix-instantiate output"
Left e -> throwIO $ FatalError ("Couldn't parse nix-instantiate output" <> show e)
-- | Run with docker-compose.yaml tmpfile
withEvaluatedComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r
withEvaluatedComposition ea f = do
v <- evaluateComposition ea
withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path handle -> do
T.hPutStrLn handle (pretty v)
hClose handle
withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path yamlHandle -> do
T.hPutStrLn yamlHandle (pretty v)
hClose yamlHandle
f path
@ -117,15 +113,14 @@ buildComposition outLink ea = do
case exitCode of
ExitSuccess -> pass
ExitFailure 1 -> exitFailure
e@ExitFailure {} -> do
ExitFailure {} -> do
throwIO $ FatalError $ "nix-build failed with " <> show exitCode
exitWith e
-- | Do something with a docker-compose.yaml.
withBuiltComposition :: EvaluationArgs -> (FilePath -> IO r) -> IO r
withBuiltComposition ea f = do
withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path handle -> do
hClose handle
withTempFile "." ".tmp-arion-docker-compose.yaml" $ \path emptyYamlHandle -> do
hClose emptyYamlHandle
-- Known problem: kills atomicity of withTempFile; won't fix because we should manage gc roots,
-- impl of which will probably avoid this "problem". It seems unlikely to cause issues.
Directory.removeFile path
@ -149,9 +144,8 @@ replForComposition ea = do
case exitCode of
ExitSuccess -> pass
ExitFailure 1 -> exitFailure
e@ExitFailure {} -> do
ExitFailure {} -> do
throwIO $ FatalError $ "nix repl failed with " <> show exitCode
exitWith e
argArgs :: EvaluationArgs -> [[Char]]
argArgs ea =

View file

@ -10,13 +10,9 @@ import Protolude hiding (to)
import qualified Data.Aeson as Aeson
import Arion.Aeson (decodeFile)
import qualified Data.ByteString as BS
import qualified System.Process as Process
import Control.Lens
import Data.Aeson.Lens
import Data.String
import System.IO (withFile, IOMode(ReadMode))
-- | Subject to change
getDefaultExec :: FilePath -> Text -> IO [Text]

View file

@ -6,16 +6,11 @@ where
import Protolude
import Test.Hspec
import Test.QuickCheck
import qualified Data.List.NonEmpty as NEL
import Arion.Aeson
import Arion.Nix
import qualified Data.Text as T
import qualified Data.Text.IO as T
import qualified Data.Text.Lazy.IO as TL
import qualified Data.Text.Lazy.Builder as TB
import qualified Data.Aeson.Encode.Pretty
import Data.Char (isSpace)
spec :: Spec
spec = describe "evaluateComposition" $ it "matches an example" $ do
@ -32,8 +27,8 @@ spec = describe "evaluateComposition" $ it "matches an example" $ do
expected <- T.readFile "src/haskell/testdata/Arion/NixSpec/arion-compose.json"
censorPaths actual `shouldBe` censorPaths expected
censorPaths :: Text -> Text
censorPaths = censorImages . censorStorePaths
--censorPaths = censorStorePaths
censorStorePaths :: Text -> Text
censorStorePaths x = case T.breakOn "/nix/store/" x of
@ -61,4 +56,4 @@ isNixNameChar c | c >= 'A' && c <= 'Z' = True
isNixNameChar c | c == '-' = True
isNixNameChar c | c == '.' = True
isNixNameChar c | c == '_' = True -- WRONG?
isNixNameChar c = False -- WRONG?
isNixNameChar _ = False -- WRONG?

View file

@ -1,5 +1,6 @@
module Main where
import Prelude()
import Protolude
import Test.Hspec.Runner
import qualified Spec