Arion: Add Grafana+Loki log aggregation

hosts/franz/arion/default.nix

@@ -21,6 +21,7 @@
     ./headscale
     ./auth
     ./minio
+    ./stats
   ];
 
   environment.systemPackages = with pkgs; [arion];

hosts/franz/arion/stats/arion-compose.nix

@@ -0,0 +1,58 @@
+{
+  project.name = "stats";
+
+  networks.dmz = {
+    name = "dmz";
+    external = true;
+  };
+  networks.internal = {};
+
+  services = {
+    grafana.service = {
+      image = "grafana/grafana:10.4.4";
+      user = "1000";
+      container_name = "grafana";
+      labels = {
+        "traefik.enable" = "true";
+
+        "traefik.http.services.grafana.loadbalancer.server.port" = "3000";
+        "traefik.http.routers.grafana.service" = "grafana";
+        "traefik.http.routers.grafana.rule" = "Host(`grafana.ghoscht.com`)";
+        "traefik.http.routers.grafana.entrypoints" = "websecure";
+        "traefik.http.routers.grafana.tls" = "true";
+        "traefik.http.routers.grafana.tls.certresolver" = "letsencrypt";
+      };
+      volumes = [
+        "/storage/dataset/docker/stats/grafana_data:/var/lib/grafana"
+      ];
+      networks = [
+        "dmz"
+        "internal"
+      ];
+    };
+    loki.service = {
+      image = "grafana/loki:3.0.0";
+      volumes = [
+        "/storage/dataset/docker/stats/loki_data:/etc/loki"
+      ];
+      ports = [
+        "3100:3100"
+      ];
+      command = "-config.file=/etc/loki/loki-config.yml";
+      networks = [
+        "internal"
+      ];
+    };
+    promtail.service = {
+      image = "grafana/promtail:3.0.0";
+      volumes = [
+        "/var/log:/var/log"
+        "/storage/dataset/docker/stats/promtail_data:/etc/promtail"
+      ];
+      command = "-config.file=/etc/promtail/promtail-config.yml";
+      networks = [
+        "internal"
+      ];
+    };
+  };
+}

hosts/franz/arion/stats/arion-pkgs.nix

@@ -0,0 +1,6 @@
+# Instead of pinning Nixpkgs, we can opt to use the one in NIX_PATH
+import <nixpkgs> {
+  # We specify the architecture explicitly. Use a Linux remote builder when
+  # calling arion from other platforms.
+  system = "x86_64-linux";
+}

hosts/franz/arion/stats/default.nix

@@ -0,0 +1,38 @@
+{config, ...}: let
+  vars = import ../../../../vars.nix;
+in {
+  virtualisation.arion = {
+    projects.stats.settings = {
+      imports = [./arion-compose.nix];
+    };
+  };
+
+  systemd.services.add-loki-logging-driver = {
+    description = "Add grafana loki docker driver";
+    after = ["network.target"];
+    wantedBy = ["multi-user.target"];
+
+    serviceConfig.Type = "oneshot";
+    script = let
+      dockercli = "${config.virtualisation.docker.package}/bin/docker";
+    in ''
+      # Put a true at the end to prevent getting non-zero return code, which will
+      # crash the whole service.
+      check=$(${dockercli} plugin ls | grep "loki" || true)
+      if [ -z "$check" ]; then
+        ${dockercli} plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
+      else
+        echo "loki docker driver already exists in docker"
+      fi
+    '';
+  };
+
+  virtualisation.docker.daemon.settings = {
+    debug = true;
+    log-driver = "loki";
+    log-opts = {
+      loki-url = "http://localhost:3100/loki/api/v1/push";
+      # loki-url = "http://host.docker.internal:3100/loki/api/v1/push";
+    };
+  };
+}