Monitoring stack (almost done)

modules/monitoring/default.nix

@@ -1 +1,7 @@
-{ imports = [ ./prometheus.nix ]; }
+{
+  imports = [
+    ./grafana.nix
+    ./loki.nix
+    ./prometheus.nix
+  ];
+}

modules/monitoring/grafana.nix

@@ -0,0 +1,130 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+with lib;
+
+let
+  cfg = config.my.monitoring.grafana;
+
+in
+{
+  options.my.monitoring.grafana = {
+    enable = lib.mkEnableOption "Enable grafana as a data visualization";
+
+    proxy = {
+      enable = lib.mkEnableOption "Set the proxy entry for this service";
+
+      domain = lib.mkOption {
+        default = "example.com";
+        type = lib.types.str;
+        description = ''
+          The domain where Caddy is reachable
+        '';
+      };
+
+      host = lib.mkOption {
+        default = "localhost";
+        type = lib.types.str;
+        description = ''
+          Host name where Grafana is running
+        '';
+      };
+
+    };
+
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.enable {
+
+      age.secrets = {
+        grafana-admin-pwd = {
+          file = ../../secrets/grafana-admin-pwd.age;
+          owner = "grafana";
+        };
+      };
+
+      services = {
+
+        grafana = {
+          enable = true;
+          settings = {
+            analytics.reporting_enabled = false;
+            database = {
+              user = "grafana";
+              type = "postgres";
+
+              host = "/run/postgresql/";
+              name = "grafana";
+            };
+            security = {
+              admin_user = "pazpi";
+              admin_password = "$__file{${config.age.secrets.grafana-admin-pwd.path}}";
+            };
+            server = {
+              domain = "grafana.neon-dory.ts.net";
+              http_addr = "0.0.0.0";
+              http_port = 3000;
+              # root_url = "https://grafana.${cfg.proxy.domain}";
+              enable_gzip = true;
+            };
+            users = {
+              default_theme = "light";
+              allow_sign_up = false;
+            };
+          };
+          # XXX Just for future reference
+          # provision.dashboards.settings.providers = [
+          #   {
+          #     name = "example";
+          #     options.path = ./dashboards/example.json;
+          #   }
+          # ];
+        };
+
+        grafana-image-renderer = {
+          enable = true;
+          provisionGrafana = true;
+          chromium = pkgs.ungoogled-chromium;
+        };
+
+        postgresql = {
+          enable = true;
+          ensureDatabases = [ "grafana" ];
+          ensureUsers = [
+            {
+              name = "grafana";
+              ensureDBOwnership = true;
+            }
+          ];
+        };
+
+      };
+
+      networking.firewall.allowedTCPPorts = [ 3000 ];
+
+    })
+
+    (lib.mkIf (cfg.proxy != { }) {
+      services.caddy = with cfg.proxy; {
+        virtualHosts."grafana.${domain}".extraConfig = ''
+          reverse_proxy http://${host}:3000
+          import cloudflare
+        '';
+      };
+    })
+  ];
+
+}
+
+# {
+#   name = "Alertmanager";
+#   type = "alertmanager";
+#   url = "http://nuc:9093";
+#   jsonData.implementation = "prometheus";
+#   jsonData.handleGrafanaManagedAlerts = true;
+# }

modules/monitoring/loki.nix

@@ -0,0 +1,94 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+with lib;
+let
+  cfg = config.my.monitoring.loki;
+in
+{
+  options.my.monitoring.loki = {
+    enable = mkEnableOption "Enable Loki log aggregation module";
+  };
+
+  config = mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server.http_listen_port = 3100;
+        server.log_level = "warn";
+
+        common = {
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore.store = "inmemory";
+          };
+          replication_factor = 1;
+          path_prefix = config.services.loki.dataDir;
+        };
+
+        schema_config = {
+          configs = [
+            {
+              from = "2024-07-01";
+              store = "tsdb";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+
+        storage_config = {
+          filesystem.directory = "${config.services.loki.dataDir}/chunks";
+          tsdb_shipper.active_index_directory = "${config.services.loki.dataDir}/tsdb-index";
+          tsdb_shipper.cache_location = "${config.services.loki.dataDir}/tsdb-cache";
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+        };
+
+        ruler = {
+          storage = {
+            type = "local";
+            local.directory = "/tmp/rules";
+          };
+          rule_path = "/tmp/scratch";
+          alertmanager_url = "http://nuc:9093";
+          ring.kvstore.store = "inmemory";
+          enable_api = true;
+        };
+
+        query_scheduler = {
+          max_outstanding_requests_per_tenant = 2048;
+        };
+      };
+
+    };
+
+    services.grafana = {
+      provision.datasources.settings = {
+        datasources = [
+          {
+            name = "Loki localhost";
+            url = "http://localhost:3100";
+            type = "loki";
+          }
+        ];
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 3100 ];
+
+  };
+
+}

modules/monitoring/prometheus.nix

@@ -30,7 +30,7 @@ in
         default = "localhost";
         type = lib.types.str;
         description = ''
-          Host name where the download manager stack is running
+          Host name where the Prometheus is running
         '';
       };
 
@@ -58,9 +58,26 @@ in
               }
             ];
           }
+          {
+            job_name = "caddy";
+            static_configs = [ { targets = [ "caddy.internal:2024" ]; } ];
+          }
         ];
       };
 
+      services.grafana = {
+        provision.datasources.settings = {
+          datasources = [
+            {
+              name = "Prometheus localhost";
+              url = "http://localhost:9090";
+              type = "prometheus";
+              isDefault = true;
+            }
+          ];
+        };
+      };
+
       networking.firewall.allowedTCPPorts = [ 9090 ];
 
     })