feat(terraform): support for remote terraform state, introduce managed redis and secrets manager

infrastructure/.gitignore

@@ -1,7 +1,8 @@
 **/cr-secret.yaml
 **/customer-values.yaml
 **/auth
-
+**/.backend.hcl
+**/kubeconfig.yaml
 **/*.lock.*
 
 auth

infrastructure/terraform/README.md

@@ -92,6 +92,69 @@ terraform output -json | jq -r .cluster_name.value
 - **Security**: The sa_key.json file contains sensitive credentials and should never be committed to version control
 - **State Management**: Consider using remote state storage for team environments
 
+## Using the bucket as a Terraform S3 backend (optional)
+
+If you want Terraform state to be stored in the object storage bucket, add a backend block to your root module.
+Note: backend blocks cannot reference resources, so you must hardcode or pass the values via variables/partials.
+
+### Bootstrap script (recommended)
+
+Note: `backend "s3" {}` is already defined in `main.tf`. The bootstrap step still works because it runs `terraform init -backend=false`, which ignores the backend block.
+
+Use the helper script to bootstrap the backend in two phases:
+1) Run a local-only apply to create the bucket + credentials.
+2) Generate `.backend.hcl` and migrate state to S3.
+
+```bash
+./scripts/init-backend.sh
+```
+
+This writes `infrastructure/terraform/.backend.hcl` (contains credentials) and runs `terraform init -force-copy`.
+You can re-run the script at any time; it reuses the existing backend config if present.
+If you want remote state from the start, run this script before your first full `terraform apply`.
+
+If you want non-interactive bootstrap:
+
+```bash
+BOOTSTRAP_AUTO_APPROVE=1 ./scripts/init-backend.sh
+```
+
+Manual phase 1 (if you want to see the exact commands the script runs):
+
+```bash
+terraform init -backend=false
+terraform apply \
+  -target=stackit_objectstorage_bucket.tfstate \
+  -target=stackit_objectstorage_credentials_group.rag_creds_group \
+  -target=stackit_objectstorage_credential.rag_creds
+```
+
+### Manual backend block
+
+```hcl
+terraform {
+  backend "s3" {
+    bucket = "<BUCKET_NAME>"
+    key    = "terraform.tfstate"
+    region = "eu01"
+
+    # Use the same credentials as above
+    access_key = "<ACCESS_KEY>"
+    secret_key = "<SECRET_KEY>"
+
+    endpoints = {
+      s3 = "https://object.storage.eu01.onstackit.cloud"
+    }
+
+    # AWS-specific checks must be disabled for STACKIT
+    skip_credentials_validation = true
+    skip_region_validation      = true
+    skip_s3_checksum            = true
+    skip_requesting_account_id  = true
+  }
+}
+```
+
 ## Cleanup
 
 To destroy all resources:

infrastructure/terraform/dns.tf

@@ -1,7 +1,9 @@
 resource "stackit_dns_zone" "rag_zone" {
-  project_id = var.project_id
-  name       = "${var.name_prefix}-zone"
-  dns_name   = var.dns_name
+  project_id    = var.project_id
+  name          = "${var.name_prefix}-zone"
+  dns_name      = var.dns_name
+  contact_email = "data-ai@stackit.cloud"
+  type          = "primary"
 }
 
 output "dns_nameservers" {

infrastructure/terraform/main.tf

@@ -1,4 +1,5 @@
 terraform {
+  backend "s3" {}
   required_providers {
     stackit = {
       source  = "stackitcloud/stackit"
@@ -9,4 +10,5 @@ terraform {
 
 provider "stackit" {
   service_account_key_path = "sa_key.json"
+  default_region           = "eu01"
 }

infrastructure/terraform/model_serving.tf

@@ -0,0 +1,12 @@
+resource "stackit_modelserving_token" "rag_modelserving" {
+  project_id = var.project_id
+  name       = "${var.name_prefix}-modelserving-token"
+
+  # No ttl_duration set -> token does not expire.
+}
+
+output "model_serving_bearer_token" {
+  description = "Bearer token for AI Model Serving API"
+  value       = stackit_modelserving_token.rag_modelserving.token
+  sensitive   = true
+}

infrastructure/terraform/object_storage.tf

@@ -1,8 +1,19 @@
+# This resource stays stable for 365 days, then changes
+resource "time_rotating" "key_rotation" {
+  rotation_days = 365
+}
+
 resource "stackit_objectstorage_bucket" "documents" {
   name       = "${var.name_prefix}-documents-${var.deployment_timestamp}"
   project_id = var.project_id
 }
 
+resource "stackit_objectstorage_bucket" "tfstate" {
+  name       = "${var.name_prefix}-tfstate-${var.deployment_timestamp}"
+  project_id = var.project_id
+  depends_on = [stackit_objectstorage_credentials_group.rag_creds_group]
+}
+
 resource "stackit_objectstorage_bucket" "langfuse" {
   name       = "${var.name_prefix}-langfuse-${var.deployment_timestamp}"
   project_id = var.project_id
@@ -16,7 +27,7 @@ resource "stackit_objectstorage_credentials_group" "rag_creds_group" {
 resource "stackit_objectstorage_credential" "rag_creds" {
   project_id           = var.project_id
   credentials_group_id = stackit_objectstorage_credentials_group.rag_creds_group.credentials_group_id
-  expiration_timestamp = timeadd(timestamp(), "8760h") # Expires after 1 year
+  expiration_timestamp = timeadd(time_rotating.key_rotation.rfc3339, "8760h")
 }
 
 output "object_storage_access_key" {
@@ -30,5 +41,5 @@ output "object_storage_secret_key" {
 }
 
 output "object_storage_bucket" {
-  value = stackit_objectstorage_bucket.documents.name
+  value = stackit_objectstorage_bucket.tfstate.name
 }

infrastructure/terraform/redis.tf

@@ -0,0 +1,18 @@
+resource "stackit_redis_instance" "rag_redis" {
+  project_id = var.project_id
+  name       = "${var.name_prefix}-redis"
+  version    = var.redis_version
+  plan_name  = var.redis_plan_name
+
+  parameters = {
+    sgw_acl                 = join(",", stackit_ske_cluster.rag_cluster.egress_address_ranges)
+    enable_monitoring       = false
+    down_after_milliseconds = 30000
+  }
+}
+
+
+resource "stackit_redis_credential" "rag_redis_cred" {
+  project_id  = var.project_id
+  instance_id = stackit_redis_instance.rag_redis.instance_id
+}

infrastructure/terraform/scripts/init-backend.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+root_dir="$(cd "${script_dir}/.." && pwd)"
+
+backend_config_file="${BACKEND_CONFIG_FILE:-${root_dir}/.backend.hcl}"
+auto_approve="${BOOTSTRAP_AUTO_APPROVE:-0}"
+
+cd "${root_dir}"
+
+if ! command -v terraform >/dev/null 2>&1; then
+  echo "terraform is not installed or not in PATH." >&2
+  exit 1
+fi
+
+if [ -f "${backend_config_file}" ]; then
+  terraform init -backend-config="${backend_config_file}"
+  exit 0
+fi
+
+echo "Bootstrapping object storage for Terraform state (local backend)."
+terraform init -backend=false
+
+if ! bucket="$(terraform output -raw object_storage_bucket 2>/dev/null)"; then
+  apply_args=(
+    "-target=stackit_objectstorage_bucket.tfstate"
+    "-target=stackit_objectstorage_credentials_group.rag_creds_group"
+    "-target=stackit_objectstorage_credential.rag_creds"
+    "-target=time_rotating.key_rotation"       # <--- Add this (needed for creds)
+    "-target=output.object_storage_bucket"     # <--- Add this
+    "-target=output.object_storage_access_key" # <--- Add this
+    "-target=output.object_storage_secret_key" # <--- Add this
+  )
+  if [ "${auto_approve}" = "1" ]; then
+    terraform apply -auto-approve "${apply_args[@]}"
+  else
+    terraform apply "${apply_args[@]}"
+  fi
+  bucket="$(terraform output -raw object_storage_bucket)"
+fi
+
+access_key="$(terraform output -raw object_storage_access_key)"
+secret_key="$(terraform output -raw object_storage_secret_key)"
+
+cat > "${backend_config_file}" <<EOF
+bucket = "${bucket}"
+key    = "terraform.tfstate"
+region = "eu01"
+
+access_key = "${access_key}"
+secret_key = "${secret_key}"
+
+endpoints = {
+  s3 = "https://object.storage.eu01.onstackit.cloud"
+}
+
+skip_credentials_validation = true
+skip_region_validation      = true
+skip_s3_checksum            = true
+skip_requesting_account_id  = true
+EOF
+
+chmod 600 "${backend_config_file}"
+
+terraform init -backend-config="${backend_config_file}" -force-copy

infrastructure/terraform/secretsmanager.tf

@@ -0,0 +1,24 @@
+resource "stackit_secretsmanager_instance" "rag_secrets" {
+  project_id = var.project_id
+  name       = "${var.name_prefix}-secrets"
+}
+
+resource "stackit_secretsmanager_user" "rag_secrets_user" {
+  project_id    = var.project_id
+  instance_id   = stackit_secretsmanager_instance.rag_secrets.instance_id
+  description   = var.secretsmanager_user_description
+  write_enabled = var.secretsmanager_user_write_enabled
+}
+
+output "secretsmanager_instance_id" {
+  value = stackit_secretsmanager_instance.rag_secrets.instance_id
+}
+
+output "secretsmanager_username" {
+  value = stackit_secretsmanager_user.rag_secrets_user.username
+}
+
+output "secretsmanager_password" {
+  value     = stackit_secretsmanager_user.rag_secrets_user.password
+  sensitive = true
+}

infrastructure/terraform/seed-secrets/README.md

@@ -0,0 +1,31 @@
+# Seed Secrets Manager data with Terraform
+
+This folder writes the `rag-secrets` KV secret used by External Secrets.
+
+Why this is a separate step:
+- The Secrets Manager instance ID and user credentials are created by the main Terraform stack.
+- Provider blocks cannot depend on resources, so we pass them in via variables and run a second apply.
+
+## Steps
+
+1. Apply the main stack to create the Secrets Manager instance and user.
+2. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in the values:
+   - `vault_mount_path` is the Secrets Manager instance ID.
+   - `vault_username`/`vault_password` are from the `secretsmanager_*` outputs.
+   - `rag_secrets` should include all keys referenced by your ExternalSecret resources.
+     For the cert-manager webhook, store the service account key JSON under `STACKIT_CERT_MANAGER_SA_JSON` (use a heredoc to avoid escaping).
+3. Run:
+   ```bash
+   terraform init
+   terraform plan
+   terraform apply
+   ```
+
+## Security note
+
+All values written by `vault_kv_secret_v2` are stored in Terraform state. Use a secure backend and restrict access.
+
+## Troubleshooting
+
+If you see a 404 from `auth/token/create`, the backend does not allow child token creation.
+This module sets `skip_child_token = true` so Vault uses the login token directly.

infrastructure/terraform/seed-secrets/main.tf

@@ -0,0 +1,26 @@
+terraform {
+  required_providers {
+    vault = {
+      source  = "hashicorp/vault"
+      version = "~> 5.6"
+    }
+  }
+}
+
+provider "vault" {
+  address          = var.vault_address
+  skip_child_token = true
+
+  auth_login {
+    path = "auth/${var.vault_userpass_path}/login/${var.vault_username}"
+    parameters = {
+      password = var.vault_password
+    }
+  }
+}
+
+resource "vault_kv_secret_v2" "rag_docs" {
+  mount     = var.vault_mount_path
+  name      = var.vault_secret_name
+  data_json = jsonencode(var.rag_secrets)
+}

infrastructure/terraform/seed-secrets/terraform.tfvars.example

@@ -0,0 +1,29 @@
+vault_mount_path = "<<SECRETS_MANAGER_INSTANCE_ID>>"
+vault_username   = "<<SECRETS_MANAGER_USERNAME>>"
+vault_password   = "<<SECRETS_MANAGER_PASSWORD>>"
+
+rag_secrets = {
+  LANGFUSE_INIT_ORG_ID    = "<LANGFUSE_INIT_ORG_ID>>"
+  LANGFUSE_INIT_PROJECT_ID = "<<LANGFUSE_INIT_PROJECT_ID>>"
+  LANGFUSE_INIT_USER_EMAIL = "<<LANGFUSE_INIT_USER_EMAIL>>"
+  LANGFUSE_INIT_USER_NAME   = "<<LANGFUSE_INIT_USER_NAME>>"
+  LANGFUSE_INIT_USER_PASSWORD = "<<LANGFUSE_INIT_USER_PASSWORD>>"
+  LANGFUSE_PUBLIC_KEY      = "<<LANGFUSE_PUBLIC_KEY>>"
+  LANGFUSE_SECRET_KEY      = "<<LANGFUSE_SECRET_KEY>>"
+  LANGFUSE_SALT            = "<<LANGFUSE_SALT>>"
+  LANGFUSE_NEXTAUTH        = "<<LANGFUSE_NEXTAUTH>>"
+  STACKIT_EMBEDDER_API_KEY = "<<STACKIT_EMBEDDER_API_KEY>>"
+  STACKIT_VLLM_API_KEY     = "<<STACKIT_VLLM_API_KEY>>"
+  RAGAS_OPENAI_API_KEY     = "<<RAGAS_OPENAI_API_KEY>>"
+  S3_ACCESS_KEY_ID         = "<<S3_ACCESS_KEY_ID>>"
+  S3_SECRET_ACCESS_KEY     = "<<S3_SECRET_ACCESS_KEY>>"
+  BASIC_AUTH_USER          = "<<BASIC_AUTH_USER>>"
+  BASIC_AUTH_PASSWORD      = "<<BASIC_AUTH_PASSWORD>>"
+  BASIC_AUTH               = "<<BASIC_AUTH_HTPASSWD>>"
+  POSTGRES_PASSWORD        = "<<POSTGRES_PASSWORD>>"
+  REDIS_PASSWORD           = "<<REDIS_PASSWORD>>"
+  CLICKHOUSE_PASSWORD      = "<<CLICKHOUSE_PASSWORD>>"
+  STACKIT_CERT_MANAGER_SA_JSON = <<EOF
+<<STACKIT_CERT_MANAGER_SA_JSON>>
+EOF
+}

infrastructure/terraform/seed-secrets/variables.tf

@@ -0,0 +1,39 @@
+variable "vault_address" {
+  description = "Vault address (STACKIT Secrets Manager URL)."
+  type        = string
+  default     = "https://prod.sm.eu01.stackit.cloud"
+}
+
+variable "vault_mount_path" {
+  description = "Secrets Manager instance ID (KV mount path)."
+  type        = string
+}
+
+variable "vault_userpass_path" {
+  description = "Vault userpass auth path."
+  type        = string
+  default     = "userpass"
+}
+
+variable "vault_username" {
+  description = "Secrets Manager user name."
+  type        = string
+}
+
+variable "vault_password" {
+  description = "Secrets Manager user password."
+  type        = string
+  sensitive   = true
+}
+
+variable "vault_secret_name" {
+  description = "KV secret name used by External Secrets."
+  type        = string
+  default     = "rag-secrets"
+}
+
+variable "rag_secrets" {
+  description = "Map of secret keys/values stored under the rag-secrets secret."
+  type        = map(string)
+  sensitive   = true
+}