# LiteLLM Gateway Configuration — Layer 1 of 2-Layer Architecture
# Deployed on CT 116 (192.168.68.116) alongside custom router on :9000
# Last updated: 2026-06-16

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  # database_url: using DATABASE_URL env var instead
  store_model_in_db: true

model_list:
  # Content-based auto-routing (router picks GPU via 5-tier analysis)
  - model_name: syslog-auto
    litellm_params:
      model: openai/syslog-auto
      api_base: http://router:9000/v1
      api_key: os.environ/ROUTER_API_KEY
      rpm: 600

  # Individual GPU strict passthrough (exact GPU, no silent fallback)
  - model_name: qwen3.6-35B-A3B
    litellm_params:
      model: openai/qwen3.6-35B-A3B
      api_base: http://router:9000/v1
      api_key: os.environ/ROUTER_API_KEY

  - model_name: qwen3.6-27B-code
    litellm_params:
      model: openai/qwen3.6-27B-code
      api_base: http://router:9000/v1
      api_key: os.environ/ROUTER_API_KEY

  - model_name: gemma-4-12b
    litellm_params:
      model: openai/gemma-4-12b
      api_base: http://router:9000/v1
      api_key: os.environ/ROUTER_API_KEY

# Guardrails: Pre-call and post-call content moderation
guardrails:
  - guardrail_name: "input-moderation"
    litellm_params:
      guardrail: openai_moderation
      mode: "pre_call"

  - guardrail_name: "output-moderation"
    litellm_params:
      guardrail: openai_moderation
      mode: "post_call"

  - guardrail_name: "harmful-content-filter"
    litellm_params:
      guardrail: litellm_content_filter
      mode: "pre_call"
      categories:
        - category: "harmful_self_harm"
          enabled: true
          action: "BLOCK"
          severity_threshold: "medium"
        - category: "harmful_violence"
          enabled: true
          action: "BLOCK"
          severity_threshold: "medium"
        - category: "harmful_illegal_weapons"
          enabled: true
          action: "BLOCK"
          severity_threshold: "medium"

litellm_settings:
  num_retries: 0                     # Disabled — our router handles retry logic
  request_timeout: 600               # Match 10-min llama-server timeout
  set_verbose: true
  failure_callback: ["prometheus"]   # Export metrics to Prometheus

router_settings:
  routing_strategy: "usage-based-routing"  # For external models only
  enable_loadbalancing_on_proxy: false     # Disable LiteLLM internal LB
  allowed_fails: 100                       # Router returns 503 on saturated GPUs
  # Fallback chains: LiteLLM retries down the chain when router returns saturated.
  # This gives accurate per-model metrics because router no longer silently reroutes.
  # The router's circuit breaker prevents cascading failures to dead GPUs.
  fallbacks:
    - syslog-auto: ["qwen3.6-35B-A3B", "qwen3.6-27B-code", "gemma-4-12b"]
    - qwen3.6-35B-A3B: ["qwen3.6-27B-code", "gemma-4-12b"]
    - qwen3.6-27B-code: ["qwen3.6-35B-A3B", "gemma-4-12b"]
    - gemma-4-12b: ["qwen3.6-27B-code", "qwen3.6-35B-A3B"]

# Cost tracking for spend analytics (local GPUs at $0, symbolic rates optional)
litellm_settings:
  model_cost:
    syslog-auto:
      input_cost_per_token: 0.0
      output_cost_per_token: 0.0
    qwen3.6-35B-A3B:
      input_cost_per_token: 0.0
      output_cost_per_token: 0.0
    qwen3.6-27B-code:
      input_cost_per_token: 0.0
      output_cost_per_token: 0.0
    gemma-4-12b:
      input_cost_per_token: 0.0
      output_cost_per_token: 0.0
    # For internal cost allocation, set symbolic rates:
    # e.g., MoE = $2/M tokens, Dense = $1/M tokens, VLM = $0.50/M tokens

# SSO/OIDC Configuration
litellm_settings:
  sso_callback: "/sso/callback"