diff --git a/hosts/caddy/default.nix b/hosts/caddy/default.nix index 515f774..7db886a 100644 --- a/hosts/caddy/default.nix +++ b/hosts/caddy/default.nix @@ -173,7 +173,12 @@ in } { subdomain = "ai"; - host = "http://${p.hosts.portainer}:4000"; + host = "http://${p.hosts.portainer}:4080"; + domain = p.domains.public; + } + { + subdomain = "keep"; + host = "http://${p.hosts.portainer}:3000"; domain = p.domains.public; } ]; diff --git a/hosts/default.nix b/hosts/default.nix index 3451afa..2a417cf 100644 --- a/hosts/default.nix +++ b/hosts/default.nix @@ -249,7 +249,7 @@ in modules = [ myModules proxmoxModule - ./open-webui + ./open-webui/docker.nix agenix.nixosModules.default ]; # specialArgs = { }; diff --git a/hosts/open-webui/default.nix b/hosts/open-webui/default.nix index fd9542e..7c99e2f 100644 --- a/hosts/open-webui/default.nix +++ b/hosts/open-webui/default.nix @@ -1,22 +1,157 @@ -{ - config, - pkgs, - lib, - ... -}: +{ config, pkgs, ... }: let - p = import ../parameters.nix; - litellm-port = 12345; + litellmSettings = { + + general_settings = { + proxy_batch_write_at = 60; # Batch write spend updates every 60s + database_connection_pool_limit = 10; # Limit the number of database connections + disable_spend_logs = true; # Turn off writing each transaction to the DB + disable_error_logs = true; # Turn off writing LLM exceptions to DB + allow_requests_on_db_unavailable = true; # Allow requests if DB is unavailable + }; + + environment_variables = { + LITELLM_MODE = "Production"; + }; + + credential_list = [ + { + credential_name = "dp_azure_openai_credential"; + credential_values = { + api_base = "os.environ/AZURE_API_BASE_OPENAI"; + api_key = "os.environ/AZURE_API_KEY_OPENAI"; + }; + credential_info = { + description = "Azure OpenAI credentials for DP"; + }; + } + { + credential_name = "dp_azure_ai_credential"; + credential_values = { + api_base = "os.environ/AZURE_API_BASE_AI"; + api_key = "os.environ/AZURE_API_KEY_AI"; + }; + credential_info = { + description = "Azure AI credentials for DP"; + }; + } + ]; + + model_list = [ + { + model_name = "text-embedding-3-large"; + litellm_params = { + model = "azure/text-embedding-3-large"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "embedding"; + }; + } + { + model_name = "text-embedding-3-small"; + litellm_params = { + model = "azure/text-embedding-3-small"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "embedding"; + }; + } + { + model_name = "GPT-3.5 Turbo"; + litellm_params = { + model = "azure/gpt-35-turbo"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4o"; + litellm_params = { + model = "azure/gpt-4o"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4.1"; + litellm_params = { + model = "azure/gpt-4.1"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4.1 Mini"; + litellm_params = { + model = "azure/gpt-4.1-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT o3 Mini"; + litellm_params = { + model = "azure/o3-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4o Mini"; + litellm_params = { + model = "azure/gpt-4o-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "Dall-e 3"; + litellm_params = { + model = "azure/dall-e-3"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "image_generation"; + }; + } + { + model_name = "azure-openai-4o-audio"; + litellm_params = { + litellm_credential_name = "dp_azure_openai_credential"; + model = "azure/gpt-4o-audio-preview"; + }; + } + { + model_name = "DeepSeek-R1"; + litellm_params = { + litellm_credential_name = "dp_azure_ai_credential"; + model = "azure_ai/deepseek-r1"; + }; + } + ]; + }; in { age.secrets = { - azure-ai.file = ../../secrets/azure-ai.age; - # open-webui-secrets.file = ../../secrets/open-webui-secrets.age; + open-webui.file = ../../secrets/open-webui.age; }; my = { + services.open-webui = { + enable = true; + port = 4000; + environmentSecretsPath = config.age.secrets.open-webui.path; + environment = { + OAUTH_PROVIDER_NAME = "authentik"; + OPENID_PROVIDER_URL = "https://auth.pasetto.me/application/o/openwebui/.well-known/openid-configuration"; + OPENID_REDIRECT_URI = "https://ai.pasetto.me/oauth/oidc/callback"; + ENABLE_OAUTH_SIGNUP = "true"; + ENABLE_LOGIN_FORM = "false"; + }; + litellm = { + enable = true; + settings = litellmSettings; + }; + }; + utils = { commons.enable = true; commons.gc.enable = true; @@ -26,74 +161,5 @@ in virtualisation.proxmox.enable = true; }; - services.litellm = { - enable = true; - environmentFile = config.age.secrets.azure-ai.path; - host = "0.0.0.0"; - openFirewall = true; - port = litellm-port; - settings = { - model_list = [ - { - model_name = "azure-gpt-35-turbo"; - litellm_params = { - model = "azure/gpt-35-turbo"; - api_base = "os.environ/AZURE_API_BASE_OPENAI"; - api_key = "os.environ/AZURE_API_KEY_OPENAI"; - }; - } - { - model_name = "azure-gpt-4o"; - litellm_params = { - model = "azure/gpt-4o"; - api_base = "os.environ/AZURE_API_BASE_OPENAI"; - api_key = "os.environ/AZURE_API_KEY_OPENAI"; - }; - } - { - model_name = "azure-gpt-4o-mini"; - litellm_params = { - model = "azure/gpt-4o-mini"; - api_base = "os.environ/AZURE_API_BASE_OPENAI"; - api_key = "os.environ/AZURE_API_KEY_OPENAI"; - }; - } - { - model_name = "azure-gpt-o3-mini"; - litellm_params = { - model = "azure/o3-mini"; - api_base = "os.environ/AZURE_API_BASE_OPENAI"; - api_key = "os.environ/AZURE_API_KEY_OPENAI"; - api_version = "2024-12-01-preview"; - }; - } - { - model_name = "azure-openai-4o-audio"; - litellm_params = { - model = "azure/gpt-4o-audio-preview"; - api_base = "os.environ/AZURE_API_BASE_OPENAI"; - api_key = "os.environ/AZURE_API_KEY_OPENAI"; - }; - } - { - model_name = "dall-e-3"; - litellm_params = { - model = "azure/dall-e-3"; - api_base = "os.environ/AZURE_API_BASE_OPENAI"; - api_key = "os.environ/AZURE_API_KEY_OPENAI"; - }; - } - { - model_name = "azure-DeepSeek-R1"; - litellm_params = { - model = "azure_ai/DeepSeek-R1"; - api_base = "os.environ/AZURE_API_BASE_R1"; - api_key = "os.environ/AZURE_API_KEY_R1"; - }; - } - ]; - }; - }; - system.stateVersion = "24.11"; } diff --git a/hosts/open-webui/docker.nix b/hosts/open-webui/docker.nix new file mode 100644 index 0000000..cb4e87f --- /dev/null +++ b/hosts/open-webui/docker.nix @@ -0,0 +1,136 @@ +{ config, pkgs, ... }: +let + litellmSettings = { + + general_settings = { + proxy_batch_write_at = 60; # Batch write spend updates every 60s + database_connection_pool_limit = 10; # Limit the number of database connections + disable_spend_logs = true; # Turn off writing each transaction to the DB + disable_error_logs = true; # Turn off writing LLM exceptions to DB + allow_requests_on_db_unavailable = true; # Allow requests if DB is unavailable + }; + + environment_variables = { + LITELLM_MODE = "Production"; + }; + + credential_list = [ + { + credential_name = "dp_azure_openai_credential"; + credential_values = { + api_base = "os.environ/AZURE_API_BASE_OPENAI"; + api_key = "os.environ/AZURE_API_KEY_OPENAI"; + }; + credential_info = { + description = "Azure OpenAI credentials for DP"; + }; + } + { + credential_name = "dp_azure_ai_credential"; + credential_values = { + api_base = "os.environ/AZURE_API_BASE_AI"; + api_key = "os.environ/AZURE_API_KEY_AI"; + }; + credential_info = { + description = "Azure AI credentials for DP"; + }; + } + ]; + + model_list = [ + { + model_name = "text-embedding-3-large"; + litellm_params = { + model = "azure/text-embedding-3-large"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "embedding"; + }; + } + { + model_name = "text-embedding-3-small"; + litellm_params = { + model = "azure/text-embedding-3-small"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "embedding"; + }; + } + { + model_name = "GPT-3.5 Turbo"; + litellm_params = { + model = "azure/gpt-35-turbo"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4o"; + litellm_params = { + model = "azure/gpt-4o"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT o3 Mini"; + litellm_params = { + model = "azure/o3-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4o Mini"; + litellm_params = { + model = "azure/gpt-4o-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "Dall-e 3"; + litellm_params = { + model = "azure/dall-e-3"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "image_generation"; + }; + } + { + model_name = "azure-openai-4o-audio"; + litellm_params = { + litellm_credential_name = "dp_azure_openai_credential"; + model = "azure/gpt-4o-audio-preview"; + }; + } + { + model_name = "DeepSeek-R1"; + litellm_params = { + litellm_credential_name = "dp_azure_ai_credential"; + model = "azure_ai/deepseek-r1"; + }; + } + ]; + }; +in +{ + + age.secrets = { + open-webui.file = ../../secrets/open-webui.age; + }; + + my = { + + virtualisation.docker.enable = true; + + utils = { + commons.enable = true; + commons.gc.enable = true; + lxc-standard.enable = true; + }; + + virtualisation.proxmox.enable = true; + }; + + system.stateVersion = "24.11"; +} diff --git a/hosts/open-webui/service.nix b/hosts/open-webui/service.nix new file mode 100644 index 0000000..045086d --- /dev/null +++ b/hosts/open-webui/service.nix @@ -0,0 +1,145 @@ +{ + config, + pkgs, + lib, + ... +}: +let + p = import ../parameters.nix; + litellm-port = 12345; +in +{ + + age.secrets = { + azure-ai.file = ../../secrets/azure-ai.age; + }; + + my = { + + utils = { + commons.enable = true; + commons.gc.enable = true; + lxc-standard.enable = true; + }; + + virtualisation.proxmox.enable = true; + }; + + services.litellm = { + enable = true; + environmentFile = config.age.secrets.azure-ai.path; + host = "0.0.0.0"; + openFirewall = true; + port = litellm-port; + settings = { + + general_settings = { + proxy_batch_write_at = 60; # Batch write spend updates every 60s + }; + + environment_variables = { + LITELLM_MODE = "Production"; + }; + + credential_list = [ + { + credential_name = "dp_azure_openai_credential"; + credential_values = { + api_base = "os.environ/AZURE_API_BASE_OPENAI"; + api_key = "os.environ/AZURE_API_KEY_OPENAI"; + }; + credential_info = { + description = "Azure OpenAI credentials for DP"; + }; + } + { + credential_name = "dp_azure_ai_credential"; + credential_values = { + api_base = "os.environ/AZURE_API_BASE_AI"; + api_key = "os.environ/AZURE_API_KEY_AI"; + }; + credential_info = { + description = "Azure AI credentials for DP"; + }; + } + ]; + + model_list = [ + { + model_name = "text-embedding-3-large"; + litellm_params = { + model = "azure/text-embedding-3-large"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "embedding"; + }; + } + { + model_name = "text-embedding-3-small"; + litellm_params = { + model = "azure/text-embedding-3-small"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "embedding"; + }; + } + { + model_name = "GPT-3.5 Turbo"; + litellm_params = { + model = "azure/gpt-35-turbo"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4o"; + litellm_params = { + model = "azure/gpt-4o"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT o3 Mini"; + litellm_params = { + model = "azure/o3-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "GPT-4o Mini"; + litellm_params = { + model = "azure/gpt-4o-mini"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + } + { + model_name = "Dall-e 3"; + litellm_params = { + model = "azure/dall-e-3"; + litellm_credential_name = "dp_azure_openai_credential"; + }; + model_info = { + mode = "image_generation"; + }; + } + { + model_name = "azure-openai-4o-audio"; + litellm_params = { + litellm_credential_name = "dp_azure_openai_credential"; + model = "azure/gpt-4o-audio-preview"; + }; + } + { + model_name = "DeepSeek-R1"; + litellm_params = { + litellm_credential_name = "dp_azure_ai_credential"; + model = "azure_ai/deepseek-r1"; + }; + } + ]; + }; + }; + + system.stateVersion = "24.11"; +} diff --git a/modules/services/default.nix b/modules/services/default.nix index 897ac6f..aaf4512 100644 --- a/modules/services/default.nix +++ b/modules/services/default.nix @@ -11,5 +11,6 @@ ./postgres.nix ./searx.nix ./vaultwarden.nix + ./open-webui.nix ]; } diff --git a/modules/services/open-webui.nix b/modules/services/open-webui.nix new file mode 100644 index 0000000..4aa460d --- /dev/null +++ b/modules/services/open-webui.nix @@ -0,0 +1,203 @@ +{ + lib, + config, + pkgs, + ... +}: +let + cfg = config.my.services.open-webui; + dbPort = 5432; + settingsFormat = pkgs.formats.yaml { }; + inherit (lib) types; +in +{ + + options.my.services.open-webui = { + + enable = lib.mkEnableOption "Enable Open Webui, alternative OpenAI frontend module"; + + port = lib.mkOption { + type = lib.types.int; + default = 8080; + description = '' + The port on which the Open Webui service will listen. + ''; + }; + + environmentSecretsPath = lib.mkOption { + type = lib.types.path; + default = ""; + description = '' + Path to the environment file containing secrets. + ''; + }; + + environment = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = "Environment variables to set for Open Webui"; + }; + + litellm = { + + enable = lib.mkEnableOption "Enable LiteLLM OpenAI proxy module"; + + port = lib.mkOption { + type = lib.types.int; + default = 12345; + description = '' + The port on which the LiteLLM service will listen. + ''; + }; + + settings = lib.mkOption { + type = types.submodule { + freeformType = settingsFormat.type; + options = { + model_list = lib.mkOption { + type = settingsFormat.type; + description = '' + List of supported models on the server, with model-specific configs. + ''; + default = [ ]; + }; + router_settings = lib.mkOption { + type = settingsFormat.type; + description = '' + LiteLLM Router settings + ''; + default = { }; + }; + + litellm_settings = lib.mkOption { + type = settingsFormat.type; + description = '' + LiteLLM Module settings + ''; + default = { }; + }; + + general_settings = lib.mkOption { + type = settingsFormat.type; + description = '' + LiteLLM Server settings + ''; + default = { }; + }; + + environment_variables = lib.mkOption { + type = settingsFormat.type; + description = '' + Environment variables to pass to the Lite + ''; + default = { }; + }; + }; + }; + default = { }; + description = '' + Configuration for LiteLLM. + See for more. + ''; + }; + }; + + }; + + config = lib.mkMerge [ + (lib.mkIf cfg.enable { + # Enable Podman as the container runtime + virtualisation.podman = { + enable = true; + autoPrune = { + enable = true; + flags = [ "--all" ]; + }; + }; + + virtualisation.oci-containers = { + backend = "podman"; + containers = { + open-webui = { + image = "ghcr.io/open-webui/open-webui:main"; + ports = [ + "${toString cfg.port}:8080" + ]; + environmentFiles = [ cfg.environmentSecretsPath ]; + environment = + cfg.environment + // { + ENABLE_OPENAI_API = "true"; + ENABLE_OLLAMA_API = "false"; + ENABLE_WEB_SEARCH = "true"; + WEB_SEARCH_ENGINE = "searxng"; + SEARXNG_QUERY_URL = "https://search.pasetto.me/search?q="; + } + // (lib.optionalAttrs cfg.litellm.enable { + OPENAI_API_BASE_URL = "http://host.containers.internal:${toString cfg.litellm.port}"; + }); + volumes = [ "open-webui:/app/backend/data" ]; + labels = { + "io.containers.autoupdate" = "registry"; + }; + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ cfg.port ]; + + }) + + (lib.mkIf cfg.litellm.enable { + + virtualisation.oci-containers.containers = { + + litellm = + let + litellmSettings = cfg.litellm.settings; + configFile = settingsFormat.generate "config.yaml" litellmSettings; + in + { + image = "ghcr.io/berriai/litellm:main-stable"; + volumes = [ "${configFile}:/app/config.yaml" ]; + cmd = [ "--config=/app/config.yaml" ]; + ports = [ "${toString cfg.litellm.port}:4000" ]; + environmentFiles = [ cfg.environmentSecretsPath ]; + environment = { + DATABASE_URL = "postgresql://llmproxy:llmproxypwd@host.containers.internal:5432/litellm"; + STORE_MODEL_IN_DB = "True"; + USE_PRISMA_MIGRATE = "True"; + }; + labels = { + "io.containers.autoupdate" = "registry"; + }; + }; + + litellm_db = { + image = "docker.io/library/postgres:16"; + hostname = "db"; + ports = [ "5432:5432" ]; + environment = { + POSTGRES_DB = "litellm"; + POSTGRES_USER = "llmproxy"; + POSTGRES_PASSWORD = "llmproxypwd"; + }; + volumes = [ "litellm_postgres_data:/var/lib/postgresql/data" ]; + labels = { + "io.containers.autoupdate" = "registry"; + }; + }; + }; + + networking.firewall.allowedTCPPorts = [ + cfg.litellm.port + dbPort + ]; + + systemd.services."podman-litellm".after = [ "podman-litellm_db.service" ]; + systemd.services."podman-litellm".requires = [ "podman-litellm_db.service" ]; + + }) + + ]; +} diff --git a/secrets.nix b/secrets.nix index b17b568..e1ce26b 100644 --- a/secrets.nix +++ b/secrets.nix @@ -29,7 +29,7 @@ let shadowsocks-password = [ machines.shadowsocks ]; firefly-iii-app-key = [ machines.firefly-iii ]; firefly-iii-mailgun-key = [ machines.firefly-iii ]; - azure-ai = [ machines.open-webui ]; + open-webui = [ machines.open-webui ]; paperless-admin = [ machines.paperless ]; paperless-oauth2-client-secret = [ machines.paperless ]; }; diff --git a/secrets/azure-ai.age b/secrets/azure-ai.age deleted file mode 100644 index 4c6757a..0000000 --- a/secrets/azure-ai.age +++ /dev/null @@ -1,17 +0,0 @@ -age-encryption.org/v1 --> ssh-ed25519 p3OXOQ /ktFoPayT7BOflXXgi24MKwAUUsZfM7fFf/Fscr/5k0 -RLe5EkY/3tZcXKwUgiWcGyhlAADN8Js/uyZKZAxUMpI --> ssh-ed25519 Si3UKw W+2Nl/Xej2KcE0gk1psQck/ndnAUO1xDbYUkAVI56l8 -gCjrHvi0tSUD8vJKvjIpfAfEIxrGpVp6ggBJK+3pXls --> ssh-ed25519 3UG3uw r+A2Dp/x7PD1+yiXubBzeGqawnriGE+ez56g+UvwGy4 -95lN581nC8mK2FlLLN6LXAh6CVF1xwYD8/YxgU5heR4 --> ssh-ed25519 JEhtoQ VTCjzgKW4lxUHXlVAQtrpyCYc7W1vZ2jlrlbq1pxAxI -vTg/4tg4Tu1ZRJFzOcW+5P8UBqaAVS2xk9Zb5xto8sI --> ssh-ed25519 uqg2jw YKVIE2E842PeN0JGkIm6JWIYkOWeHC0zw7gNZU6aCjg -7W83czFxv++ygFF1RcOELvQBR8ZiN1j9cEWo/iiFdEM ---- 6Gv+howHUPpUS7voypuS6jiw/CqWqYACa/dRTCaoWYY -j]Hj,:XU_ *z6yS:!-'y“6ġ\5Xy\}j Ee%@˔=Ͼ'az03cOgV&A4BCPi$U{[>Mz|Zمʉ l X>[݁˓Hxk.TwQutWVTG,"1ۃu׼Zj#N\z˖уT4˽6$qtx -mv5籚)pn[E~:.^ 6 D4~hE\' -\W -Qâlv$K5W`Sx? \C6 -{D/m$is2v \ No newline at end of file diff --git a/secrets/open-webui.age b/secrets/open-webui.age new file mode 100644 index 0000000..29d8f98 Binary files /dev/null and b/secrets/open-webui.age differ diff --git a/ssh-keys.nix b/ssh-keys.nix index a0dfa61..b648e43 100644 --- a/ssh-keys.nix +++ b/ssh-keys.nix @@ -21,7 +21,7 @@ rec { dns02 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ+HIq6/ebjiv71xDozdOTn5AdnXgr1fGqIzXnH7Not+"; shadowsocks = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQ4qYaS5ccciH7BNyrF5+J3d4JtHJNr1R256/ulEtxl"; firefly-iii = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYkXjRqmuTMg56EmAx8s1M/VQojM7akF/ao+jJLYgFB"; - open-webui = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIhpg3rWdLaV+rInpUggwOB7F09cREH9ffeR5z8eZXD9"; + open-webui = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAOkm9z19sFGEs7aexOfnvyxEgehydSbeLjrYo0srFKV"; paperless = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILRNgDyk3TuMooG4ZCv7SOgXh0ql1/1hhhng7uSnsLeK"; };