From 73376a0a7d46b342f11a07e76bf3d50991e7db2b Mon Sep 17 00:00:00 2001 From: Danijel Simeunovic Date: Thu, 30 Apr 2026 22:38:33 +0200 Subject: [PATCH] vault migration --- README.md | 36 ++- .../argocd-mcp-credentials-vault.yaml | 14 ++ apps/base/argo-mcp/auth-oidc-vault.yaml | 14 ++ apps/base/argo-mcp/kustomization.yaml | 6 +- apps/base/argo-mcp/vault-auth.yaml | 20 ++ .../dot-ai-stack/dot-ai-secrets-vault.yaml | 14 ++ apps/base/dot-ai-stack/kustomization.yaml | 4 +- apps/base/dot-ai-stack/vault-auth.yaml | 20 ++ apps/base/mcp10x/app-credentials-vault.yaml | 15 ++ apps/base/mcp10x/kustomization.yaml | 4 +- apps/base/mcp10x/vault-auth.yaml | 20 ++ apps/base/musicman/kustomization.yaml | 4 +- .../musicman/musicman-credentials-vault.yaml | 15 ++ apps/base/musicman/vault-auth.yaml | 20 ++ apps/base/ts-mcp/kustomization.yaml | 4 +- apps/base/ts-mcp/ts-mcp-secrets-vault.yaml | 14 ++ apps/base/ts-mcp/vault-auth.yaml | 20 ++ .../argocd-notifications-secret-vault.yaml | 15 ++ cluster-resources/forte-helm-repo-vault.yaml | 16 ++ .../forte10x-repo-credentials-vault.yaml | 17 ++ .../mcp10x-repo-credentials-vault.yaml | 17 ++ cluster-resources/vault-auth-argocd.yaml | 20 ++ docs/DEVELOPER-GUIDE.md | 208 +++++++----------- docs/OPERATIONS-RUNBOOK.md | 174 +++++---------- docs/vault-secrets-operator.md | 206 +++++++++++++++++ infra/base/gitea/gitea-backup-s3-vault.yaml | 15 ++ infra/base/gitea/gitea-credentials-vault.yaml | 14 ++ .../base/gitea/gitea-runner-token-vault.yaml | 14 ++ infra/base/gitea/gitea-smtp-secret-vault.yaml | 17 ++ infra/base/gitea/kustomization.yaml | 10 +- infra/base/gitea/vault-auth.yaml | 20 ++ .../homepage-widget-credentials-vault.yaml | 14 ++ infra/base/homepage/kustomization.yaml | 4 +- infra/base/homepage/vault-auth.yaml | 20 ++ .../keycloak/keycloak-credentials-vault.yaml | 14 ++ infra/base/keycloak/kustomization.yaml | 4 +- infra/base/keycloak/vault-auth.yaml | 20 ++ infra/base/kustomization.yaml | 1 + infra/base/renovate/kustomization.yaml | 4 +- infra/base/renovate/renovate-env-vault.yaml | 17 ++ infra/base/renovate/vault-auth.yaml | 20 ++ infra/base/sealedsecrets/kustomization.yaml | 2 +- .../vault-secrets-operator/kustomization.yaml | 4 + .../vault-secrets-operator.yaml | 42 ++++ infra/overlays/upc-dev/kustomization.yaml | 3 +- .../microsoft-idp-credentials-vault.yaml | 14 ++ .../base/vault-secrets-operator-values.yaml | 19 ++ scripts/seed-vault-from-cluster.sh | 85 +++++++ scripts/vault-setup-policies.sh | 81 +++++++ 49 files changed, 1103 insertions(+), 272 deletions(-) create mode 100644 apps/base/argo-mcp/argocd-mcp-credentials-vault.yaml create mode 100644 apps/base/argo-mcp/auth-oidc-vault.yaml create mode 100644 apps/base/argo-mcp/vault-auth.yaml create mode 100644 apps/base/dot-ai-stack/dot-ai-secrets-vault.yaml create mode 100644 apps/base/dot-ai-stack/vault-auth.yaml create mode 100644 apps/base/mcp10x/app-credentials-vault.yaml create mode 100644 apps/base/mcp10x/vault-auth.yaml create mode 100644 apps/base/musicman/musicman-credentials-vault.yaml create mode 100644 apps/base/musicman/vault-auth.yaml create mode 100644 apps/base/ts-mcp/ts-mcp-secrets-vault.yaml create mode 100644 apps/base/ts-mcp/vault-auth.yaml create mode 100644 cluster-resources/argocd-notifications-secret-vault.yaml create mode 100644 cluster-resources/forte-helm-repo-vault.yaml create mode 100644 cluster-resources/forte10x-repo-credentials-vault.yaml create mode 100644 cluster-resources/mcp10x-repo-credentials-vault.yaml create mode 100644 cluster-resources/vault-auth-argocd.yaml create mode 100644 docs/vault-secrets-operator.md create mode 100644 infra/base/gitea/gitea-backup-s3-vault.yaml create mode 100644 infra/base/gitea/gitea-credentials-vault.yaml create mode 100644 infra/base/gitea/gitea-runner-token-vault.yaml create mode 100644 infra/base/gitea/gitea-smtp-secret-vault.yaml create mode 100644 infra/base/gitea/vault-auth.yaml create mode 100644 infra/base/homepage/homepage-widget-credentials-vault.yaml create mode 100644 infra/base/homepage/vault-auth.yaml create mode 100644 infra/base/keycloak/keycloak-credentials-vault.yaml create mode 100644 infra/base/keycloak/vault-auth.yaml create mode 100644 infra/base/renovate/renovate-env-vault.yaml create mode 100644 infra/base/renovate/vault-auth.yaml create mode 100644 infra/base/vault-secrets-operator/kustomization.yaml create mode 100644 infra/base/vault-secrets-operator/vault-secrets-operator.yaml create mode 100644 infra/overlays/upc-dev/microsoft-idp-credentials-vault.yaml create mode 100644 infra/values/base/vault-secrets-operator-values.yaml create mode 100644 scripts/seed-vault-from-cluster.sh create mode 100644 scripts/vault-setup-policies.sh diff --git a/README.md b/README.md index 8b300fe..f3fbdc6 100644 --- a/README.md +++ b/README.md @@ -57,11 +57,11 @@ This repository contains the complete GitOps configuration for our Kubernetes cl ### What's Inside -- **Infrastructure Applications**: Traefik, Cert-Manager, Kyverno, Prometheus, Grafana, Loki, Tempo, Sealed Secrets, Homepage (platform dashboard) +- **Infrastructure Applications**: Traefik, Cert-Manager, Kyverno, Prometheus, Grafana, Loki, Tempo, Vault, Vault Secrets Operator, Homepage (platform dashboard) - **Business Applications**: MCP10X, MusicMan, Dot-AI Stack, ArgoCD MCP - **Policies**: Kyverno security policies for secret management, namespace controls, pod verification - **Monitoring**: Full observability stack with metrics, logs, traces, and alerting -- **Secrets**: Sealed Secrets for secure Git storage +- **Secrets**: Vault Secrets Operator (VSO) syncs secrets from HashiCorp Vault to K8s ### Key Features @@ -187,7 +187,7 @@ Developer commits code → CI/CD builds image → Updates helm-prod-values → A **Quick version**: 1. Create `apps/myapp.yaml` (ArgoCD Application manifest) 2. Create `helm-prod-values/myapp/values.yaml` (configuration) -3. Create sealed secrets if needed +3. Write secrets to Vault and create VaultStaticSecret CRD if needed 4. Commit and push - ArgoCD auto-syncs! ### Update an Existing Application @@ -200,22 +200,18 @@ Developer commits code → CI/CD builds image → Updates helm-prod-values → A ### Manage Secrets -**See detailed guide**: [Developer Guide - Working with Secrets](docs/DEVELOPER-GUIDE.md#working-with-secrets) +**See detailed guide**: [Vault Secrets Operator Reference](docs/vault-secrets-operator.md) ```bash -# Create plain secret -kubectl create secret generic myapp-creds \ - --from-literal=KEY=value \ - --dry-run=client -o yaml > private/myapp-creds.yaml +# 1. Write secret to Vault +vault kv put kv/myapp/myapp-creds KEY=value -# Seal it -kubeseal --format=yaml --cert=pub-cert.pem \ - < private/myapp-creds.yaml > secrets/myapp-creds-sealed.yaml +# 2. Create VaultStaticSecret CRD (one-time, commit to git) +# See docs/vault-secrets-operator.md for CRD template -# Commit sealed version -git add secrets/myapp-creds-sealed.yaml -git commit -m "Add myapp credentials" -git push +# 3. Rotate secrets — no git commit needed! +vault kv put kv/myapp/myapp-creds KEY=new-value +# VSO picks up changes within 30 seconds ``` ### Enable Authentication @@ -328,7 +324,7 @@ kubectl patch application myapp -n argocd \ ## 🔐 Security ### Secret Management -- ✅ Sealed Secrets for Git storage +- ✅ Vault Secrets Operator (VSO) for secret management - ✅ Kyverno auto-clones secrets to namespaces - ❌ Never commit plain secrets @@ -355,7 +351,8 @@ kubectl patch application myapp -n argocd \ | **Traefik** | Ingress controller | `traefik` | 2 | | **Cert-Manager** | TLS certificates | `cert-manager` | 1 | | **Kyverno** | Policy engine | `kyverno` | 1 | -| **Sealed Secrets** | Secret encryption | `kube-system` | 1 | +| **Vault** | Secret storage | `vault` | 1 | +| **Vault Secrets Operator** | Secret sync (Vault → K8s) | `vault-secrets-operator-system` | 1 | | **Prometheus** | Metrics | `monitoring` | 1 | | **Grafana** | Dashboards | `monitoring` | 1 | | **Loki** | Logs | `monitoring` | 1 | @@ -455,7 +452,7 @@ Applications deploy in order using `argocd.argoproj.io/sync-wave`: 1. Read [Developer Guide - Deploying Your First Application](docs/DEVELOPER-GUIDE.md#deploying-your-first-application) 2. Create ArgoCD Application manifest in `apps/` 3. Create Helm values in `helm-prod-values/` -4. Create sealed secrets if needed +4. Write secrets to Vault and create VaultStaticSecret CRD if needed 5. Commit and push - ArgoCD handles the rest! ### Modifying Infrastructure @@ -499,7 +496,8 @@ Documentation lives in `docs/`. To update: - [Traefik Documentation](https://doc.traefik.io/traefik/) - [Cert-Manager Documentation](https://cert-manager.io/docs/) - [Grafana Tempo Documentation](https://grafana.com/docs/tempo/) -- [Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets) +- [Vault Secrets Operator](https://developer.hashicorp.com/vault/docs/platform/k8s/vso) +- [HashiCorp Vault](https://developer.hashicorp.com/vault/docs) ### Related Repositories - [forte-helm](https://git.forteapps.net/Forte/forte-helm) - Helm chart templates diff --git a/apps/base/argo-mcp/argocd-mcp-credentials-vault.yaml b/apps/base/argo-mcp/argocd-mcp-credentials-vault.yaml new file mode 100644 index 0000000..d745d37 --- /dev/null +++ b/apps/base/argo-mcp/argocd-mcp-credentials-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: argocd-mcp-credentials + namespace: argocd-mcp +spec: + type: kv-v2 + mount: kv + path: argocd-mcp/argocd-mcp-credentials + destination: + name: argocd-mcp-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/apps/base/argo-mcp/auth-oidc-vault.yaml b/apps/base/argo-mcp/auth-oidc-vault.yaml new file mode 100644 index 0000000..6d25184 --- /dev/null +++ b/apps/base/argo-mcp/auth-oidc-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: auth-oidc + namespace: argocd-mcp +spec: + type: kv-v2 + mount: kv + path: argocd-mcp/auth-oidc + destination: + name: auth-oidc + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/apps/base/argo-mcp/kustomization.yaml b/apps/base/argo-mcp/kustomization.yaml index 0e1bb35..29ba4c6 100644 --- a/apps/base/argo-mcp/kustomization.yaml +++ b/apps/base/argo-mcp/kustomization.yaml @@ -2,5 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - argo-mcp.yaml -- argocdmcp-auth-oidc-sealed.yaml -- argocd-mcp-credentials.yaml +- vault-auth.yaml +- auth-oidc-vault.yaml +- argocd-mcp-credentials-vault.yaml +# Removed: argocdmcp-auth-oidc-sealed.yaml, argocd-mcp-credentials.yaml (migrated to VSO) diff --git a/apps/base/argo-mcp/vault-auth.yaml b/apps/base/argo-mcp/vault-auth.yaml new file mode 100644 index 0000000..fc2aecc --- /dev/null +++ b/apps/base/argo-mcp/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-argocd-mcp + namespace: argocd-mcp +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: argocd-mcp +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-argocd-mcp + serviceAccount: vault-auth-argocd-mcp + audiences: + - vault diff --git a/apps/base/dot-ai-stack/dot-ai-secrets-vault.yaml b/apps/base/dot-ai-stack/dot-ai-secrets-vault.yaml new file mode 100644 index 0000000..b1641b3 --- /dev/null +++ b/apps/base/dot-ai-stack/dot-ai-secrets-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: dot-ai-secrets + namespace: dot-ai +spec: + type: kv-v2 + mount: kv + path: dot-ai/dot-ai-secrets + destination: + name: dot-ai-secrets + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/apps/base/dot-ai-stack/kustomization.yaml b/apps/base/dot-ai-stack/kustomization.yaml index 400e157..104ae46 100644 --- a/apps/base/dot-ai-stack/kustomization.yaml +++ b/apps/base/dot-ai-stack/kustomization.yaml @@ -2,4 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - dot-ai-stack.yaml -- dot-ai-secrets.yaml +- vault-auth.yaml +- dot-ai-secrets-vault.yaml +# Removed: dot-ai-secrets.yaml (migrated to VSO) diff --git a/apps/base/dot-ai-stack/vault-auth.yaml b/apps/base/dot-ai-stack/vault-auth.yaml new file mode 100644 index 0000000..39a6e69 --- /dev/null +++ b/apps/base/dot-ai-stack/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-dot-ai + namespace: dot-ai +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: dot-ai +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-dot-ai + serviceAccount: vault-auth-dot-ai + audiences: + - vault diff --git a/apps/base/mcp10x/app-credentials-vault.yaml b/apps/base/mcp10x/app-credentials-vault.yaml new file mode 100644 index 0000000..dbd6b84 --- /dev/null +++ b/apps/base/mcp10x/app-credentials-vault.yaml @@ -0,0 +1,15 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: app-credentials + namespace: mcp10x +spec: + type: kv-v2 + mount: kv + path: mcp10x/app-credentials + destination: + name: app-credentials + create: true + type: Opaque + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/apps/base/mcp10x/kustomization.yaml b/apps/base/mcp10x/kustomization.yaml index efda0de..42d39d3 100644 --- a/apps/base/mcp10x/kustomization.yaml +++ b/apps/base/mcp10x/kustomization.yaml @@ -2,4 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - mcp10x.yaml -- forte10x-app-credentials-sealed.yaml +- vault-auth.yaml +- app-credentials-vault.yaml +# Removed: forte10x-app-credentials-sealed.yaml (migrated to VSO) diff --git a/apps/base/mcp10x/vault-auth.yaml b/apps/base/mcp10x/vault-auth.yaml new file mode 100644 index 0000000..a5ce4b7 --- /dev/null +++ b/apps/base/mcp10x/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-mcp10x + namespace: mcp10x +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: mcp10x +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-mcp10x + serviceAccount: vault-auth-mcp10x + audiences: + - vault diff --git a/apps/base/musicman/kustomization.yaml b/apps/base/musicman/kustomization.yaml index 83d1a26..4bb7f28 100644 --- a/apps/base/musicman/kustomization.yaml +++ b/apps/base/musicman/kustomization.yaml @@ -2,4 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - musicman.yaml -- musicman-credentials.yaml +- vault-auth.yaml +- musicman-credentials-vault.yaml +# Removed: musicman-credentials.yaml (migrated to VSO) diff --git a/apps/base/musicman/musicman-credentials-vault.yaml b/apps/base/musicman/musicman-credentials-vault.yaml new file mode 100644 index 0000000..79cda80 --- /dev/null +++ b/apps/base/musicman/musicman-credentials-vault.yaml @@ -0,0 +1,15 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: musicman-credentials + namespace: music-man +spec: + type: kv-v2 + mount: kv + path: music-man/musicman-credentials + destination: + name: musicman-credentials + create: true + type: Opaque + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/apps/base/musicman/vault-auth.yaml b/apps/base/musicman/vault-auth.yaml new file mode 100644 index 0000000..f6d9f90 --- /dev/null +++ b/apps/base/musicman/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-music-man + namespace: music-man +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: music-man +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-music-man + serviceAccount: vault-auth-music-man + audiences: + - vault diff --git a/apps/base/ts-mcp/kustomization.yaml b/apps/base/ts-mcp/kustomization.yaml index 4d8206c..af9bde8 100644 --- a/apps/base/ts-mcp/kustomization.yaml +++ b/apps/base/ts-mcp/kustomization.yaml @@ -2,4 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ts-mcp.yaml -- ts-mcp-secrets-sealed.yaml +- vault-auth.yaml +- ts-mcp-secrets-vault.yaml +# Removed: ts-mcp-secrets-sealed.yaml (migrated to VSO) diff --git a/apps/base/ts-mcp/ts-mcp-secrets-vault.yaml b/apps/base/ts-mcp/ts-mcp-secrets-vault.yaml new file mode 100644 index 0000000..289a03f --- /dev/null +++ b/apps/base/ts-mcp/ts-mcp-secrets-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: ts-mcp-secrets + namespace: ts-mcp +spec: + type: kv-v2 + mount: kv + path: ts-mcp/ts-mcp-secrets + destination: + name: ts-mcp-secrets + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/apps/base/ts-mcp/vault-auth.yaml b/apps/base/ts-mcp/vault-auth.yaml new file mode 100644 index 0000000..f138764 --- /dev/null +++ b/apps/base/ts-mcp/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-ts-mcp + namespace: ts-mcp +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: ts-mcp +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-ts-mcp + serviceAccount: vault-auth-ts-mcp + audiences: + - vault diff --git a/cluster-resources/argocd-notifications-secret-vault.yaml b/cluster-resources/argocd-notifications-secret-vault.yaml new file mode 100644 index 0000000..34af912 --- /dev/null +++ b/cluster-resources/argocd-notifications-secret-vault.yaml @@ -0,0 +1,15 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: argocd-notifications-secret + namespace: argocd +spec: + type: kv-v2 + mount: kv + path: argocd/argocd-notifications-secret + destination: + name: argocd-notifications-secret + create: true + type: Opaque + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/cluster-resources/forte-helm-repo-vault.yaml b/cluster-resources/forte-helm-repo-vault.yaml new file mode 100644 index 0000000..334b1dd --- /dev/null +++ b/cluster-resources/forte-helm-repo-vault.yaml @@ -0,0 +1,16 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: forte-helm-repo + namespace: argocd +spec: + type: kv-v2 + mount: kv + path: argocd/forte-helm-repo + destination: + name: forte-helm-repo + create: true + labels: + argocd.argoproj.io/secret-type: repository + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/cluster-resources/forte10x-repo-credentials-vault.yaml b/cluster-resources/forte10x-repo-credentials-vault.yaml new file mode 100644 index 0000000..dd4f6ae --- /dev/null +++ b/cluster-resources/forte10x-repo-credentials-vault.yaml @@ -0,0 +1,17 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: forte10x-repo-creds + namespace: argocd +spec: + type: kv-v2 + mount: kv + path: argocd/forte10x-repo-creds + destination: + name: forte10x-repo-creds + create: true + type: Opaque + labels: + argocd.argoproj.io/secret-type: repository + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/cluster-resources/mcp10x-repo-credentials-vault.yaml b/cluster-resources/mcp10x-repo-credentials-vault.yaml new file mode 100644 index 0000000..79cdad7 --- /dev/null +++ b/cluster-resources/mcp10x-repo-credentials-vault.yaml @@ -0,0 +1,17 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: mcp10x-repo-creds + namespace: argocd +spec: + type: kv-v2 + mount: kv + path: argocd/mcp10x-repo-creds + destination: + name: mcp10x-repo-creds + create: true + type: Opaque + labels: + argocd.argoproj.io/secret-type: repository + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/cluster-resources/vault-auth-argocd.yaml b/cluster-resources/vault-auth-argocd.yaml new file mode 100644 index 0000000..e631385 --- /dev/null +++ b/cluster-resources/vault-auth-argocd.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-argocd + namespace: argocd +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: argocd +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-argocd + serviceAccount: vault-auth-argocd + audiences: + - vault diff --git a/docs/DEVELOPER-GUIDE.md b/docs/DEVELOPER-GUIDE.md index 77e9465..5d2a745 100644 --- a/docs/DEVELOPER-GUIDE.md +++ b/docs/DEVELOPER-GUIDE.md @@ -60,18 +60,16 @@ If you do need cluster access, install: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" ``` -2. **kubeseal** - For sealing secrets +2. **vault** CLI - For managing secrets in HashiCorp Vault ```bash # macOS - brew install kubeseal + brew install hashicorp/tap/vault # Windows - choco install kubeseal + choco install vault # Linux - wget https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.24.0/kubeseal-0.24.0-linux-amd64.tar.gz - tar -xvzf kubeseal-0.24.0-linux-amd64.tar.gz - sudo mv kubeseal /usr/local/bin/ + # See https://developer.hashicorp.com/vault/install ``` 3. **Git** - Version control @@ -634,115 +632,100 @@ git push ### Understanding Secret Management -**NEVER commit plain secrets to Git.** We use **Sealed Secrets** to encrypt secrets before committing. +Secrets are managed via **HashiCorp Vault** and synced to Kubernetes by the **Vault Secrets Operator (VSO)**. See [Vault Secrets Operator Reference](vault-secrets-operator.md) for full details. + +**NEVER commit plain secret values to Git.** Only VaultStaticSecret CRD manifests are committed. ### Creating a New Secret -#### Step 1: Create Plain Secret Locally +#### Step 1: Write Secret to Vault ```bash -cd ~/dev/k8s/launchpad - -# Create secret in private/ folder (Git-ignored) -kubectl create secret generic myapp-credentials \ - --from-literal=API_KEY=your-secret-key-here \ - --from-literal=DB_PASSWORD=super-secret-password \ - --dry-run=client -o yaml > private/myapp-credentials.yaml +vault kv put kv/myapp/myapp-credentials \ + API_KEY=your-secret-key-here \ + DB_PASSWORD=super-secret-password ``` -**DO NOT commit this file!** It's in `private/` which is Git-ignored. +#### Step 2: Create VaultStaticSecret CRD -#### Step 2: Seal the Secret +Create a YAML file (e.g., `apps/base/myapp/myapp-credentials-vault.yaml`): -Seal your secret: - -```bash -kubeseal --format=yaml \ - --namespace=myapp \ - < private/myapp-credentials.yaml \ - > secrets/myapp-credentials-sealed.yaml +```yaml +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: myapp-credentials + namespace: myapp +spec: + type: kv-v2 + mount: kv + path: myapp/myapp-credentials + destination: + name: myapp-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth ``` -#### Step 3: Commit Sealed Secret +#### Step 3: Add VaultAuth (if new namespace) + +If this is a new namespace, also create a `vault-auth.yaml` with a ServiceAccount and VaultAuth CRD. See [VSO Reference](vault-secrets-operator.md#vaultauth) for template. + +#### Step 4: Commit and Push ```bash -git add secrets/myapp-credentials-sealed.yaml -git commit -m "Add myapp credentials (sealed)" +git add apps/base/myapp/myapp-credentials-vault.yaml +git commit -m "Add myapp credentials (VSO)" git push ``` -#### Step 4: Reference Secret in Application +ArgoCD syncs the CRD, VSO creates the K8s Secret. + +#### Step 5: Reference Secret in Application Update your `helm-prod-values/myapp/values.yaml`: ```yaml app: - envSecretName: "myapp-credentials" # References the SealedSecret + envSecretName: "myapp-credentials" # VSO creates this K8s Secret ``` -Commit and push: +### Updating / Rotating a Secret + +**No git commit needed** — just update in Vault: + ```bash -cd ~/dev/k8s/helm-prod-values -git add myapp/values.yaml -git commit -m "Reference myapp credentials" -git push +vault kv put kv/myapp/myapp-credentials \ + API_KEY=new-key-here \ + DB_PASSWORD=new-password ``` -### Updating a Secret - -To update an existing secret: +VSO picks up changes within 30 seconds. Restart pods if they don't watch for secret updates: ```bash -# 1. Create new version of secret -kubectl create secret generic myapp-credentials \ - --from-literal=API_KEY=new-key-here \ - --from-literal=DB_PASSWORD=new-password \ - --dry-run=client -o yaml > private/myapp-credentials.yaml - -# 2. Seal it -kubeseal --format=yaml \ - --namespace=myapp \ - < private/myapp-credentials.yaml \ - > secrets/myapp-credentials-sealed.yaml - -# 3. Commit sealed version -git add secrets/myapp-credentials-sealed.yaml -git commit -m "Update myapp credentials" -git push - -# 4. Restart pods to pick up new secret kubectl rollout restart deployment myapp -n myapp ``` ### Secret Best Practices -✅ **DO**: -- Store secrets in `private/` folder locally -- Always seal secrets before committing -- Delete plain secrets after sealing -- Use meaningful secret names +- Write secrets to Vault via UI or CLI — never commit values to Git +- Use meaningful secret names matching the KV path convention: `kv/{namespace}/{secret-name}` - Document what each secret contains - -❌ **DON'T**: -- Commit plain secrets to Git -- Share secrets via Slack/email -- Hard-code secrets in code -- Use the same secret across multiple environments -- Store secrets in Docker images +- Use Vault's versioning for audit trail ### Where Secrets Are Stored ``` -┌─────────────────────────────────────────────────────────────┐ -│ Location │ Content │ Committed?│ -├──────────────────────────┼────────────────────┼────────────┤ -│ private/ │ Plain secrets │ ❌ NO │ -│ secrets/ │ Sealed secrets │ ✅ YES │ -│ Kubernetes cluster │ Unsealed secrets │ N/A │ -└─────────────────────────────────────────────────────────────┘ +┌──────────────────────────────────────────────────────────────────┐ +│ Location │ Content │ In Git? │ +├────────────────────────────┼─────────────────────────┼──────────┤ +│ Vault KV (kv/{ns}/{name}) │ Secret values │ ❌ NO │ +│ VaultStaticSecret CRD │ Sync config (no values)│ ✅ YES │ +│ Kubernetes cluster │ K8s Secret (synced) │ N/A │ +└──────────────────────────────────────────────────────────────────┘ ``` -**Sealed Secrets Controller** in the cluster decrypts sealed secrets automatically. +**Vault Secrets Operator** syncs secrets from Vault to K8s automatically (30s refresh). --- @@ -876,28 +859,13 @@ In your identity provider (e.g., Keycloak): #### Step 2: Create OIDC Secret ```bash -# Create plain secret -kubectl create secret generic auth-oidc \ - --from-literal=client-secret=your-oidc-client-secret \ - --from-literal=cookie-secret=$(openssl rand -hex 32) \ - --namespace=myapp \ - --dry-run=client -o yaml > private/myapp-auth-oidc.yaml +# Write OIDC secret to Vault +vault kv put kv/myapp/auth-oidc \ + client-secret=your-oidc-client-secret \ + cookie-secret=$(openssl rand -hex 32) -# Seal it -kubeseal --format=yaml \ - --cert=pub-cert.pem \ - --namespace=myapp \ - < private/myapp-auth-oidc.yaml \ - > secrets/myapp-auth-oidc-sealed.yaml - -# Commit sealed secret -cd ~/dev/k8s/launchpad -git add secrets/myapp-auth-oidc-sealed.yaml -git commit -m "Add OIDC secrets for myapp" -git push - -# Clean up -rm private/myapp-auth-oidc.yaml +# Create VaultStaticSecret CRD (see docs/vault-secrets-operator.md for template) +# Add to apps/base/myapp/auth-oidc-vault.yaml and commit ``` #### Step 3: Configure Helm Values @@ -1127,16 +1095,13 @@ ingress: host: web-app.forteapps.net ``` -**With sealed OIDC secret**: +**With Vault OIDC secret**: ```bash -# Create and seal secret -kubectl create secret generic auth-oidc \ - --from-literal=client-secret=super-secret-value \ - --from-literal=cookie-secret=$(openssl rand -hex 32) \ - --namespace=web-app \ - --dry-run=client -o yaml | \ - kubeseal --format=yaml --cert=pub-cert.pem --namespace=web-app \ - > secrets/web-app-auth-oidc-sealed.yaml +# Write OIDC secret to Vault +vault kv put kv/web-app/auth-oidc \ + client-secret=super-secret-value \ + cookie-secret=$(openssl rand -hex 32) +# Then create VaultStaticSecret CRD — see docs/vault-secrets-operator.md ``` #### Example 3: MCP Server with OAuth 2.0 @@ -1264,7 +1229,7 @@ kubectl logs -n myapp -c authn - Use token auth for service-to-service communication - Rotate tokens and secrets regularly - Use strong random tokens (32+ bytes) -- Store client secrets in SealedSecrets +- Store client secrets in Vault - Test authentication before deploying to production - Document which tokens/users have access @@ -1568,22 +1533,22 @@ curl http://localhost:8080 #### Problem: Secret not found -**Check if SealedSecret exists:** +**Check VSO sync status:** ```bash -kubectl get sealedsecret -n myapp +kubectl get vaultstaticsecret -n myapp kubectl get secret -n myapp ``` **Solutions:** ```bash -# Check if secret is in Git -ls -l secrets/myapp-credentials-sealed.yaml +# Check VaultAuth is authenticated +kubectl get vaultauth -n myapp -# Re-apply sealed secret -kubectl apply -f secrets/myapp-credentials-sealed.yaml +# Check VaultStaticSecret events +kubectl describe vaultstaticsecret myapp-credentials -n myapp -# Check sealed-secrets-controller logs -kubectl logs -n kube-system deployment/sealed-secrets-controller +# Verify secret exists in Vault +vault kv get kv/myapp/myapp-credentials ``` #### Problem: Secret exists but pods can't access it @@ -1694,7 +1659,7 @@ If you're stuck: ### Secret Management ✅ **DO**: -- Use kubeseal for all secrets +- Use Vault for all secrets (see docs/vault-secrets-operator.md) - Store plain secrets in password manager - Rotate secrets regularly - Use different secrets per environment @@ -1746,16 +1711,9 @@ kubectl rollout restart deployment myapp -n myapp # Port-forward to service kubectl port-forward -n myapp service/myapp 8080:3000 -# Create secret -kubectl create secret generic myapp-credentials \ - --from-literal=KEY=value \ - --dry-run=client -o yaml > private/myapp-credentials.yaml - -# Seal secret -kubeseal --format=yaml \ - --cert=pub-cert.pem \ - < private/myapp-credentials.yaml \ - > secrets/myapp-credentials-sealed.yaml +# Write secret to Vault +vault kv put kv/myapp/myapp-credentials KEY=value +# Create VaultStaticSecret CRD — see docs/vault-secrets-operator.md ``` ### Repository Locations diff --git a/docs/OPERATIONS-RUNBOOK.md b/docs/OPERATIONS-RUNBOOK.md index 586a806..303c224 100644 --- a/docs/OPERATIONS-RUNBOOK.md +++ b/docs/OPERATIONS-RUNBOOK.md @@ -188,13 +188,15 @@ Save the following file in private/ (gitignored) folder as secret.yaml project: default ``` -Seal the secret using `kubeseal` command +Write the secret to Vault: ```bash -kubeseal --format=yaml \ - --namespace=argocd \ - < private/secret.yaml \ - > secrets/forte-helm-repo-secret-sealed.yaml +vault kv put kv/argocd/forte-helm-repo \ + type=git \ + url=ssh://git@git.forteapps.net:2222/Forte/forte-helm.git \ + sshPrivateKey="$(cat private/ssh-key)" \ + project=default ``` +Then create a VaultStaticSecret CRD with `argocd.argoproj.io/secret-type: repository` label. **Step 4: Register Repository in ArgoCD** @@ -499,7 +501,7 @@ See [Developer Guide](DEVELOPER-GUIDE.md#deploying-your-first-application) for d **Quick checklist:** - [ ] Create `helm-prod-values/myapp/values.yaml` - [ ] Create `apps/myapp.yaml` in config repo -- [ ] Create SealedSecret if needed +- [ ] Write secrets to Vault and create VaultStaticSecret CRD if needed - [ ] Commit and push changes - [ ] Verify sync in Slack/ArgoCD - [ ] Configure DNS for domain @@ -670,92 +672,61 @@ db: ## Secret Management +Secrets are managed via **HashiCorp Vault** and synced to Kubernetes by the **Vault Secrets Operator (VSO)**. See [Vault Secrets Operator Reference](vault-secrets-operator.md) for full details. + ### Creating Secrets -#### Step 1: Get Public Certificate +#### Step 1: Write to Vault ```bash -# Fetch sealed-secrets public cert (one-time) -kubeseal --fetch-cert \ - --controller-name=sealed-secrets-controller \ - --controller-namespace=kube-system \ - > pub-cert.pem - -# Save this certificate for future use +# From literal values +vault kv put kv/myapp/myapp-credentials \ + API_KEY=secret123 \ + DB_PASSWORD=pass456 ``` -#### Step 2: Create Plain Secret +#### Step 2: Create VaultStaticSecret CRD -```bash -# Method 1: From literal values -kubectl create secret generic myapp-credentials \ - --from-literal=API_KEY=secret123 \ - --from-literal=DB_PASSWORD=pass456 \ - --namespace=myapp \ - --dry-run=client -o yaml > private/myapp-credentials.yaml - -# Method 2: From file -kubectl create secret generic myapp-credentials \ - --from-file=.env \ - --namespace=myapp \ - --dry-run=client -o yaml > private/myapp-credentials.yaml - -# Method 3: From multiple files -kubectl create secret generic myapp-credentials \ - --from-file=api-key.txt \ - --from-file=db-password.txt \ - --namespace=myapp \ - --dry-run=client -o yaml > private/myapp-credentials.yaml +```yaml +# apps/base/myapp/myapp-credentials-vault.yaml +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: myapp-credentials + namespace: myapp +spec: + type: kv-v2 + mount: kv + path: myapp/myapp-credentials + destination: + name: myapp-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth ``` -#### Step 3: Seal Secret +#### Step 3: Commit CRD ```bash -kubeseal --format=yaml \ - --cert=pub-cert.pem \ - --namespace=myapp \ - < private/myapp-credentials.yaml \ - > secrets/myapp-credentials-sealed.yaml -``` - -#### Step 4: Commit Sealed Secret - -```bash -git add secrets/myapp-credentials-sealed.yaml -git commit -m "Add myapp credentials" +git add apps/base/myapp/myapp-credentials-vault.yaml +git commit -m "Add myapp credentials (VSO)" git push - -# Delete plain secret -rm private/myapp-credentials.yaml ``` -### Updating Secrets +ArgoCD syncs the CRD, VSO creates the K8s Secret automatically. + +### Updating / Rotating Secrets + +**No git commit needed** — just update in Vault: ```bash -# 1. Create new version -kubectl create secret generic myapp-credentials \ - --from-literal=API_KEY=new-secret-key \ - --from-literal=DB_PASSWORD=new-password \ - --namespace=myapp \ - --dry-run=client -o yaml > private/myapp-credentials.yaml +vault kv put kv/myapp/myapp-credentials \ + API_KEY=new-secret-key \ + DB_PASSWORD=new-password -# 2. Seal it -kubeseal --format=yaml \ - --cert=pub-cert.pem \ - --namespace=myapp \ - < private/myapp-credentials.yaml \ - > secrets/myapp-credentials-sealed.yaml - -# 3. Commit -git add secrets/myapp-credentials-sealed.yaml -git commit -m "Update myapp credentials" -git push - -# 4. Restart pods to pick up new secret +# VSO picks up changes within 30 seconds +# Restart pods if needed kubectl rollout restart deployment myapp -n myapp - -# 5. Delete plain secret -rm private/myapp-credentials.yaml ``` ### Viewing Secrets (Unsealed) @@ -832,30 +803,13 @@ OIDC auth requires an `auth-oidc` Secret with two keys: CLIENT_SECRET="your-oidc-client-secret-from-provider" COOKIE_SECRET=$(openssl rand -hex 32) -# Create plain secret -kubectl create secret generic auth-oidc \ - --from-literal=client-secret=$CLIENT_SECRET \ - --from-literal=cookie-secret=$COOKIE_SECRET \ - --namespace=myapp \ - --dry-run=client -o yaml > private/myapp-auth-oidc.yaml +# Write to Vault +vault kv put kv/myapp/auth-oidc \ + client-secret=$CLIENT_SECRET \ + cookie-secret=$COOKIE_SECRET -# Seal it -kubeseal --format=yaml \ - --cert=pub-cert.pem \ - --namespace=myapp \ - < private/myapp-auth-oidc.yaml \ - > secrets/myapp-auth-oidc-sealed.yaml - -# Apply sealed secret -kubectl apply -f secrets/myapp-auth-oidc-sealed.yaml - -# Commit to Git -git add secrets/myapp-auth-oidc-sealed.yaml -git commit -m "Add OIDC secrets for myapp" -git push - -# Clean up -rm private/myapp-auth-oidc.yaml +# Create VaultStaticSecret CRD (one-time) and commit +# See docs/vault-secrets-operator.md for CRD template ``` #### Rotating Authentication Secrets @@ -882,16 +836,12 @@ kubectl rollout restart deployment myapp -n myapp # Rotate cookie secret (safe - invalidates existing sessions) NEW_COOKIE_SECRET=$(openssl rand -hex 32) -# Recreate secret -kubectl create secret generic auth-oidc \ - --from-literal=client-secret=$CLIENT_SECRET \ - --from-literal=cookie-secret=$NEW_COOKIE_SECRET \ - --namespace=myapp \ - --dry-run=client -o yaml | \ - kubeseal --format=yaml --cert=pub-cert.pem --namespace=myapp | \ - kubectl apply -f - +# Update in Vault — no git commit needed +vault kv put kv/myapp/auth-oidc \ + client-secret=$CLIENT_SECRET \ + cookie-secret=$NEW_COOKIE_SECRET -# Restart to pick up new secret +# VSO picks up within 30s. Restart pods to use new secret: kubectl rollout restart deployment myapp -n myapp ``` @@ -1342,13 +1292,11 @@ kubectl get applications -n argocd -w - pg_dump -U $DB_USER -d $DB_NAME > /backup/dump-$(date +%Y%m%d).sql ``` -3. **Sealed Secrets private key backup** +3. **Vault backup** ```bash - # Backup sealed-secrets controller private key - kubectl get secret -n kube-system sealed-secrets-key \ - -o yaml > sealed-secrets-key-backup.yaml - - # Store in secure location (password manager, vault) + # Vault data is stored on PVC — ensure PVC snapshots are configured + # For disaster recovery, maintain Vault unseal keys in a secure location + # All secrets can be re-seeded from source if needed ``` --- @@ -1668,7 +1616,7 @@ echo "Remember to delete: $SECRET_FILE" - [ ] Gitea Actions workflow configured - [ ] Helm values created in `helm-prod-values/` - [ ] ArgoCD application manifest created in `apps/` -- [ ] Secrets created and sealed +- [ ] Secrets written to Vault and VaultStaticSecret CRD created - [ ] DNS record added for domain - [ ] Application synced successfully - [ ] Health check passed diff --git a/docs/vault-secrets-operator.md b/docs/vault-secrets-operator.md new file mode 100644 index 0000000..8ef902e --- /dev/null +++ b/docs/vault-secrets-operator.md @@ -0,0 +1,206 @@ +# Vault Secrets Operator (VSO) Reference + +## Overview + +The platform uses HashiCorp Vault Secrets Operator (VSO) to sync secrets from Vault KV v2 to native Kubernetes Secrets. This replaces the previous SealedSecrets workflow. + +**Key benefit**: Secret values can be rotated via Vault UI/CLI without a git commit. Only new VaultStaticSecret CRDs need to be committed. + +## Architecture + +``` +Vault (KV v2) VSO K8s Secret +kv/{namespace}/{name} --> VaultStaticSecret CRD --> Secret in namespace + (polls every 30s) +``` + +- **Vault**: Standalone instance in `vault` namespace, KV v2 at `kv/` +- **VSO**: Deployed in `vault-secrets-operator-system` namespace via ArgoCD +- **Auth**: Kubernetes auth method — each namespace has its own ServiceAccount + VaultAuth CRD + +## KV Path Convention + +``` +kv/{namespace}/{secret-name} +``` + +Examples: +- `kv/homepage/homepage-widget-credentials` +- `kv/argocd/forte-helm-repo` +- `kv/gitea/gitea-smtp-secret` +- `kv/keycloak/keycloak-credentials` + +## Vault Policy Structure + +Each namespace gets a read-only policy: + +```hcl +# Policy: ns-{namespace} +path "kv/data/{namespace}/*" { + capabilities = ["read"] +} +path "kv/metadata/{namespace}/*" { + capabilities = ["read", "list"] +} +``` + +## Kubernetes Auth Roles + +Each namespace has a bound ServiceAccount: + +``` +Role: ns-{namespace} + bound_service_account_names: vault-auth-{namespace} + bound_service_account_namespaces: {namespace} + policies: ns-{namespace} + audience: vault + ttl: 1h +``` + +## CRD Reference + +### VaultAuth + +Per-namespace auth binding. One per namespace. + +```yaml +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: {namespace} +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-{namespace} + serviceAccount: vault-auth-{namespace} + audiences: + - vault +``` + +Each VaultAuth requires a corresponding ServiceAccount: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-{namespace} + namespace: {namespace} +``` + +### VaultStaticSecret + +One per secret. Syncs a Vault KV path to a K8s Secret. + +```yaml +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: {secret-name} + namespace: {namespace} +spec: + type: kv-v2 + mount: kv + path: {namespace}/{secret-name} + destination: + name: {secret-name} # K8s Secret name (must match what apps expect) + create: true + type: Opaque # Optional, defaults to Opaque + labels: # Optional, for secrets that need labels + some-label: "value" + refreshAfter: 30s + vaultAuthRef: vault-auth +``` + +## Special Labels + +Some secrets require specific labels for correct operation: + +| Secret | Label | Purpose | +|--------|-------|---------| +| `renovate-env` | `allowedToBeCloned: "true"` | Kyverno secret-cloner policy | +| `gitea-smtp-secret` | `allowedToBeCloned: "true"` | Kyverno secret-cloner policy | +| `forte-helm-repo` | `argocd.argoproj.io/secret-type: repository` | ArgoCD repository recognition | +| `forte10x-repo-creds` | `argocd.argoproj.io/secret-type: repository` | ArgoCD repository recognition | +| `mcp10x-repo-creds` | `argocd.argoproj.io/secret-type: repository` | ArgoCD repository recognition | + +These are set in `destination.labels` of the VaultStaticSecret CRD. + +## Namespaces & Secrets Map + +| Namespace | Secrets | +|-----------|---------| +| `homepage` | homepage-widget-credentials | +| `renovate` | renovate-env | +| `gitea` | gitea-credentials, gitea-backup-s3, gitea-smtp-secret, gitea-runner-token | +| `keycloak` | keycloak-credentials, microsoft-idp-credentials (overlay) | +| `argocd` | forte-helm-repo, forte10x-repo-creds, mcp10x-repo-creds, argocd-notifications-secret | +| `mcp10x` | app-credentials | +| `ts-mcp` | ts-mcp-secrets | +| `argocd-mcp` | auth-oidc, argocd-mcp-credentials | +| `dot-ai` | dot-ai-secrets | +| `music-man` | musicman-credentials | + +## Common Operations + +### Add a new secret + +1. Write to Vault: + ```bash + vault kv put kv/{namespace}/{secret-name} key1=val1 key2=val2 + ``` + +2. Create VaultStaticSecret YAML (see template above) + +3. Add to kustomization.yaml in the appropriate directory + +4. Commit and push — ArgoCD syncs the CRD, VSO creates the K8s Secret + +### Rotate a secret value + +No git commit needed: +```bash +vault kv put kv/{namespace}/{secret-name} key1=new-val1 key2=new-val2 +``` +VSO picks up changes within 30 seconds. + +### Check sync status + +```bash +# VaultAuth status +kubectl get vaultauth -n {namespace} + +# VaultStaticSecret status +kubectl get vaultstaticsecret -n {namespace} + +# Verify K8s Secret exists with correct keys +kubectl get secret {name} -n {namespace} -o jsonpath='{.data}' | jq +``` + +### Troubleshooting + +1. **VaultAuth not authenticating**: Check ServiceAccount exists, Vault role matches SA name/namespace +2. **VaultStaticSecret not syncing**: Check `kubectl describe vaultstaticsecret {name} -n {ns}` for events +3. **Secret missing keys**: Verify Vault KV path has all expected keys: `vault kv get kv/{ns}/{name}` +4. **Permission denied**: Verify Vault policy allows read on `kv/data/{ns}/*` + +## File Locations + +| Type | Location | +|------|----------| +| VSO ArgoCD Application | `infra/base/vault-secrets-operator/` | +| VSO Helm values | `infra/values/base/vault-secrets-operator-values.yaml` | +| Vault policies script | `scripts/vault-setup-policies.sh` | +| Seed script | `scripts/seed-vault-from-cluster.sh` | +| VaultAuth + VaultStaticSecret | Alongside ArgoCD Application in each component directory | + +## Setup Scripts + +```bash +# Create all Vault policies and auth roles +./scripts/vault-setup-policies.sh + +# Seed Vault KV from existing K8s Secrets +./scripts/seed-vault-from-cluster.sh +``` diff --git a/infra/base/gitea/gitea-backup-s3-vault.yaml b/infra/base/gitea/gitea-backup-s3-vault.yaml new file mode 100644 index 0000000..24187a0 --- /dev/null +++ b/infra/base/gitea/gitea-backup-s3-vault.yaml @@ -0,0 +1,15 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: gitea-backup-s3 + namespace: gitea +spec: + type: kv-v2 + mount: kv + path: gitea/gitea-backup-s3 + destination: + name: gitea-backup-s3 + create: true + type: Opaque + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/gitea/gitea-credentials-vault.yaml b/infra/base/gitea/gitea-credentials-vault.yaml new file mode 100644 index 0000000..e8cae0a --- /dev/null +++ b/infra/base/gitea/gitea-credentials-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: gitea-credentials + namespace: gitea +spec: + type: kv-v2 + mount: kv + path: gitea/gitea-credentials + destination: + name: gitea-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/gitea/gitea-runner-token-vault.yaml b/infra/base/gitea/gitea-runner-token-vault.yaml new file mode 100644 index 0000000..52e6589 --- /dev/null +++ b/infra/base/gitea/gitea-runner-token-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: gitea-runner-token + namespace: gitea +spec: + type: kv-v2 + mount: kv + path: gitea/gitea-runner-token + destination: + name: gitea-runner-token + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/gitea/gitea-smtp-secret-vault.yaml b/infra/base/gitea/gitea-smtp-secret-vault.yaml new file mode 100644 index 0000000..0941786 --- /dev/null +++ b/infra/base/gitea/gitea-smtp-secret-vault.yaml @@ -0,0 +1,17 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: gitea-smtp-secret + namespace: gitea +spec: + type: kv-v2 + mount: kv + path: gitea/gitea-smtp-secret + destination: + name: gitea-smtp-secret + create: true + type: Opaque + labels: + allowedToBeCloned: "true" + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/gitea/kustomization.yaml b/infra/base/gitea/kustomization.yaml index 256f91f..c31f520 100644 --- a/infra/base/gitea/kustomization.yaml +++ b/infra/base/gitea/kustomization.yaml @@ -2,7 +2,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - gitea.yaml -- gitea-backup-s3-sealed.yaml -- gitea-credentials-sealed.yaml -- gitea-runner-token-sealed.yaml -- gitea-smtp-secret-sealed.yaml +- vault-auth.yaml +- gitea-credentials-vault.yaml +- gitea-backup-s3-vault.yaml +- gitea-smtp-secret-vault.yaml +- gitea-runner-token-vault.yaml +# Removed: gitea-*-sealed.yaml (migrated to VSO) diff --git a/infra/base/gitea/vault-auth.yaml b/infra/base/gitea/vault-auth.yaml new file mode 100644 index 0000000..48b3de2 --- /dev/null +++ b/infra/base/gitea/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-gitea + namespace: gitea +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: gitea +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-gitea + serviceAccount: vault-auth-gitea + audiences: + - vault diff --git a/infra/base/homepage/homepage-widget-credentials-vault.yaml b/infra/base/homepage/homepage-widget-credentials-vault.yaml new file mode 100644 index 0000000..b797bb5 --- /dev/null +++ b/infra/base/homepage/homepage-widget-credentials-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: homepage-widget-credentials + namespace: homepage +spec: + type: kv-v2 + mount: kv + path: homepage/homepage-widget-credentials + destination: + name: homepage-widget-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/homepage/kustomization.yaml b/infra/base/homepage/kustomization.yaml index d2c23da..2a34da7 100644 --- a/infra/base/homepage/kustomization.yaml +++ b/infra/base/homepage/kustomization.yaml @@ -2,5 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - homepage.yaml -- homepage-widget-credentials-sealed.yaml +- vault-auth.yaml +- homepage-widget-credentials-vault.yaml - homepage-extra-rbac.yaml +# Removed: homepage-widget-credentials-sealed.yaml (migrated to VSO) diff --git a/infra/base/homepage/vault-auth.yaml b/infra/base/homepage/vault-auth.yaml new file mode 100644 index 0000000..a3ff0bc --- /dev/null +++ b/infra/base/homepage/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-homepage + namespace: homepage +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: homepage +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-homepage + serviceAccount: vault-auth-homepage + audiences: + - vault diff --git a/infra/base/keycloak/keycloak-credentials-vault.yaml b/infra/base/keycloak/keycloak-credentials-vault.yaml new file mode 100644 index 0000000..aa172c5 --- /dev/null +++ b/infra/base/keycloak/keycloak-credentials-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: keycloak-credentials + namespace: keycloak +spec: + type: kv-v2 + mount: kv + path: keycloak/keycloak-credentials + destination: + name: keycloak-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/keycloak/kustomization.yaml b/infra/base/keycloak/kustomization.yaml index 24f51fe..4f1cd9b 100644 --- a/infra/base/keycloak/kustomization.yaml +++ b/infra/base/keycloak/kustomization.yaml @@ -2,4 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - keycloak.yaml -- keycloak-credentials-sealed.yaml +- vault-auth.yaml +- keycloak-credentials-vault.yaml +# Removed: keycloak-credentials-sealed.yaml (migrated to VSO) diff --git a/infra/base/keycloak/vault-auth.yaml b/infra/base/keycloak/vault-auth.yaml new file mode 100644 index 0000000..3a397c1 --- /dev/null +++ b/infra/base/keycloak/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-keycloak + namespace: keycloak +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: keycloak +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-keycloak + serviceAccount: vault-auth-keycloak + audiences: + - vault diff --git a/infra/base/kustomization.yaml b/infra/base/kustomization.yaml index 6e802ef..e1ef018 100644 --- a/infra/base/kustomization.yaml +++ b/infra/base/kustomization.yaml @@ -23,3 +23,4 @@ resources: - databunker - homepage - vault +- vault-secrets-operator diff --git a/infra/base/renovate/kustomization.yaml b/infra/base/renovate/kustomization.yaml index bafd9d7..0c78724 100644 --- a/infra/base/renovate/kustomization.yaml +++ b/infra/base/renovate/kustomization.yaml @@ -2,4 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - renovate.yaml -- renovate-env-sealed.yaml +- vault-auth.yaml +- renovate-env-vault.yaml +# Removed: renovate-env-sealed.yaml (migrated to VSO) diff --git a/infra/base/renovate/renovate-env-vault.yaml b/infra/base/renovate/renovate-env-vault.yaml new file mode 100644 index 0000000..57e00db --- /dev/null +++ b/infra/base/renovate/renovate-env-vault.yaml @@ -0,0 +1,17 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: renovate-env + namespace: renovate +spec: + type: kv-v2 + mount: kv + path: renovate/renovate-env + destination: + name: renovate-env + create: true + type: Opaque + labels: + allowedToBeCloned: "true" + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/base/renovate/vault-auth.yaml b/infra/base/renovate/vault-auth.yaml new file mode 100644 index 0000000..34cda7a --- /dev/null +++ b/infra/base/renovate/vault-auth.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-auth-renovate + namespace: renovate +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vault-auth + namespace: renovate +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: ns-renovate + serviceAccount: vault-auth-renovate + audiences: + - vault diff --git a/infra/base/sealedsecrets/kustomization.yaml b/infra/base/sealedsecrets/kustomization.yaml index 6808461..b260def 100644 --- a/infra/base/sealedsecrets/kustomization.yaml +++ b/infra/base/sealedsecrets/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - sealedsecrets.yaml -- argocd-forte-helm-secret-sealed.yaml +# Removed: argocd-forte-helm-secret-sealed.yaml (migrated to VSO — now in cluster-resources/) diff --git a/infra/base/vault-secrets-operator/kustomization.yaml b/infra/base/vault-secrets-operator/kustomization.yaml new file mode 100644 index 0000000..06d7640 --- /dev/null +++ b/infra/base/vault-secrets-operator/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- vault-secrets-operator.yaml diff --git a/infra/base/vault-secrets-operator/vault-secrets-operator.yaml b/infra/base/vault-secrets-operator/vault-secrets-operator.yaml new file mode 100644 index 0000000..938f34a --- /dev/null +++ b/infra/base/vault-secrets-operator/vault-secrets-operator.yaml @@ -0,0 +1,42 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: vault-secrets-operator + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "2" + labels: + app.kubernetes.io/name: vault-secrets-operator + app.kubernetes.io/part-of: security + app.kubernetes.io/managed-by: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + + sources: + - repoURL: https://helm.releases.hashicorp.com + chart: vault-secrets-operator + targetRevision: "0.10.0" + helm: + releaseName: vault-secrets-operator + valueFiles: + - $values/infra/values/base/vault-secrets-operator-values.yaml + + - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git + targetRevision: HEAD + ref: values + + destination: + server: https://kubernetes.default.svc + namespace: vault-secrets-operator-system + + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=true + - Validate=true + - ServerSideApply=true diff --git a/infra/overlays/upc-dev/kustomization.yaml b/infra/overlays/upc-dev/kustomization.yaml index 14ccfc8..7175e6a 100644 --- a/infra/overlays/upc-dev/kustomization.yaml +++ b/infra/overlays/upc-dev/kustomization.yaml @@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../base -- entra-upc-dev-credentials-sealed.yaml +- microsoft-idp-credentials-vault.yaml +# Removed: entra-upc-dev-credentials-sealed.yaml (migrated to VSO) # No patches needed — base already has "upc-dev" paths # upc-dev is the default/base cluster diff --git a/infra/overlays/upc-dev/microsoft-idp-credentials-vault.yaml b/infra/overlays/upc-dev/microsoft-idp-credentials-vault.yaml new file mode 100644 index 0000000..c9313e5 --- /dev/null +++ b/infra/overlays/upc-dev/microsoft-idp-credentials-vault.yaml @@ -0,0 +1,14 @@ +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: microsoft-idp-credentials + namespace: keycloak +spec: + type: kv-v2 + mount: kv + path: keycloak/microsoft-idp-credentials + destination: + name: microsoft-idp-credentials + create: true + refreshAfter: 30s + vaultAuthRef: vault-auth diff --git a/infra/values/base/vault-secrets-operator-values.yaml b/infra/values/base/vault-secrets-operator-values.yaml new file mode 100644 index 0000000..be49cbc --- /dev/null +++ b/infra/values/base/vault-secrets-operator-values.yaml @@ -0,0 +1,19 @@ +# Vault Secrets Operator Helm values +# Docs: https://developer.hashicorp.com/vault/docs/platform/k8s/vso + +# Default Vault connection — used by VaultAuth CRDs that don't specify one +defaultVaultConnection: + enabled: true + address: http://vault.vault.svc.cluster.local:8200 + +# Default auth method — Kubernetes auth +defaultAuthMethod: + enabled: true + namespace: "" + method: kubernetes + mount: kubernetes + kubernetes: + role: vso-operator + serviceAccount: default + audiences: + - vault diff --git a/scripts/seed-vault-from-cluster.sh b/scripts/seed-vault-from-cluster.sh new file mode 100644 index 0000000..a25db7e --- /dev/null +++ b/scripts/seed-vault-from-cluster.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# seed-vault-from-cluster.sh — Read existing K8s Secrets and write to Vault KV +# +# Prerequisites: +# - vault CLI authenticated (VAULT_ADDR + VAULT_TOKEN set) +# - kubectl access to the cluster +# - KV v2 engine at kv/ +# +# Usage: ./scripts/seed-vault-from-cluster.sh +# +# This reads plaintext values from existing K8s Secrets and writes them +# to Vault KV v2 at kv/{namespace}/{secret-name}. + +set -euo pipefail + +echo "=== Seeding Vault KV from existing K8s Secrets ===" +echo "" + +# Helper: read a K8s secret and write all keys to Vault KV +seed_secret() { + local ns="$1" + local secret_name="$2" + local vault_path="kv/${ns}/${secret_name}" + + echo "--- ${ns}/${secret_name} → ${vault_path} ---" + + # Get all keys from the secret + local keys + keys=$(kubectl get secret "${secret_name}" -n "${ns}" -o json 2>/dev/null | \ + jq -r '.data // {} | keys[]' 2>/dev/null) || { + echo " SKIP: secret not found in cluster" + echo "" + return + } + + if [ -z "${keys}" ]; then + echo " SKIP: no data keys" + echo "" + return + fi + + # Build vault kv put arguments + local args=() + for key in ${keys}; do + local value + value=$(kubectl get secret "${secret_name}" -n "${ns}" -o jsonpath="{.data.${key}}" | base64 -d) + args+=("${key}=${value}") + done + + vault kv put "${vault_path}" "${args[@]}" + echo " OK: $(echo "${keys}" | wc -w | tr -d ' ') keys written" + echo "" +} + +# --- Homepage --- +seed_secret homepage homepage-widget-credentials + +# --- Renovate --- +seed_secret renovate renovate-env + +# --- Gitea --- +seed_secret gitea gitea-credentials +seed_secret gitea gitea-backup-s3 +seed_secret gitea gitea-smtp-secret +seed_secret gitea gitea-runner-token + +# --- Keycloak --- +seed_secret keycloak keycloak-credentials +seed_secret keycloak microsoft-idp-credentials + +# --- ArgoCD --- +seed_secret argocd forte-helm-repo +seed_secret argocd forte10x-repo-creds +seed_secret argocd mcp10x-repo-creds +seed_secret argocd argocd-notifications-secret + +# --- Application secrets --- +seed_secret mcp10x app-credentials +seed_secret ts-mcp ts-mcp-secrets +seed_secret argocd-mcp auth-oidc +seed_secret argocd-mcp argocd-mcp-credentials +seed_secret dot-ai dot-ai-secrets +seed_secret music-man musicman-credentials + +echo "=== Done. Verify with: vault kv list kv/{namespace} ===" diff --git a/scripts/vault-setup-policies.sh b/scripts/vault-setup-policies.sh new file mode 100644 index 0000000..963e0cd --- /dev/null +++ b/scripts/vault-setup-policies.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# vault-setup-policies.sh — Create Vault policies + Kubernetes auth roles for VSO +# +# Prerequisites: +# - vault CLI authenticated (VAULT_ADDR + VAULT_TOKEN set) +# - Kubernetes auth method enabled at auth/kubernetes/ +# - KV v2 secrets engine at kv/ +# +# Usage: ./scripts/vault-setup-policies.sh + +set -euo pipefail + +echo "=== Vault Secrets Operator — Policy & Auth Role Setup ===" +echo "" + +# All namespaces that have secrets to migrate +NAMESPACES=( + argocd + gitea + keycloak + renovate + homepage + argocd-mcp + mcp10x + ts-mcp + dot-ai + music-man + vault-secrets-operator-system +) + +# --- Per-namespace policies and auth roles --- + +for NS in "${NAMESPACES[@]}"; do + echo "--- Namespace: ${NS} ---" + + # Create read-only policy for this namespace's secrets + echo " Creating policy: ns-${NS}" + vault policy write "ns-${NS}" - <