feature/multicluster

Co-authored-by: Danijel Simeunovic <danijel.simeunovic@trumf.no> Reviewed-on: #4 Reviewed-by: gitea_admin <admin@forteapps.net>
2026-04-18 18:14:00 +00:00
parent 72a65f0e06
commit 03a0d7c9ae
72 changed files with 574 additions and 326 deletions
--- a/docs/DEVELOPER-GUIDE.md
+++ b/docs/DEVELOPER-GUIDE.md
@@ -1364,7 +1364,7 @@ Existing clients (like Gitea) are defined directly in `forte-realm.json` inside

 #### Step 1: Add Client to Realm Config

-In `infra/values/keycloak-values.yaml`, add a new entry to the `clients` array in `forte-realm.json`:
+In `infra/values/base/keycloak-values.yaml`, add a new entry to the `clients` array in `forte-realm.json`:

 ```json
 {
@@ -1404,7 +1404,7 @@ existingSecret: myapp-oidc-credentials

 ```bash
 cd ~/dev/k8s/launchpad
-git add infra/values/keycloak-values.yaml
+git add infra/values/base/keycloak-values.yaml
 git commit -m "Add myapp Keycloak client with auto-sync"
 git push
 ```
--- a/docs/GITOPS-ARCHITECTURE.md
+++ b/docs/GITOPS-ARCHITECTURE.md
@@ -16,7 +16,7 @@ This Kubernetes cluster uses a **GitOps approach** powered by **ArgoCD**, where

 ### Key Characteristics
 - **Environment**: Production (internal use only)
- **Cluster Type**: Single cluster, single environment
+- **Cluster Type**: Multi-cluster (upc-dev, upc-prod) via Kustomize overlays
 - **GitOps Tool**: ArgoCD
 - **Deployment Pattern**: App-of-Apps
 - **Secret Management**: Sealed Secrets (kubeseal)
@@ -62,8 +62,8 @@ This Kubernetes cluster uses a **GitOps approach** powered by **ArgoCD**, where
                                                        │
                                                        ▼
                                           ┌────────────────────────────────┐
-                                           │   Kubernetes Cluster           │
-                                           │   (UpCloud Managed)            │
+                                           │   Kubernetes Clusters          │
+                                           │   (UpCloud: upc-dev, upc-prod) │
                                           │                                │
                                           │  ┌──────────────────────────┐  │
                                           │  │    ArgoCD                │  │
@@ -116,74 +116,68 @@ This Kubernetes cluster uses a **GitOps approach** powered by **ArgoCD**, where
 ```
 launchpad/
 ├── bootstrap.sh                      # Cluster initialization script
-├── _app-of-apps.yaml                 # Root ArgoCD Application (App-of-Apps pattern)
+├── _app-of-apps-upc-dev.yaml        # Root ArgoCD Application (upc-dev cluster)
+├── _app-of-apps-upc-prod.yaml       # Root ArgoCD Application (upc-prod cluster)
 │
-├── infra/                            # Infrastructure ArgoCD Applications
-│   ├── enterprise-apps.yaml          # Parent app managing all apps in apps/
-│   ├── cluster-resources-application.yaml
-│   ├── traefik-application.yaml
-│   ├── cert-manager-application.yaml
-│   ├── kyverno.yaml
-│   ├── kyverno-policies.yaml
-│   ├── prometheus.yaml
-│   ├── grafana.yaml
-│   ├── loki.yaml
-│   ├── tempo.yaml
-│   ├── fluent-bit.yaml
-│   ├── trivy.yaml
-│   ├── sealedsecrets.yaml
-│   ├── secrets.yaml
+├── infra/                            # Infrastructure ArgoCD Applications (Kustomize)
+│   ├── base/                         # Base Application manifests (upc-dev defaults)
+│   │   ├── kustomization.yaml
+│   │   ├── traefik-application.yaml
+│   │   ├── keycloak.yaml
+│   │   ├── grafana.yaml
+│   │   ├── gitea.yaml
+│   │   ├── gitea-actions.yaml
+│   │   ├── tempo.yaml
+│   │   ├── renovate.yaml
+│   │   ├── ...                       # All other Application manifests
+│   │   └── secrets.yaml
+│   ├── overlays/                     # Per-cluster overrides
+│   │   ├── upc-dev/                  # UpCloud Dev (uses base as-is)
+│   │   └── upc-prod/                # UpCloud Prod (patches value paths)
+│   ├── dashboards/                   # Grafana dashboard ConfigMaps
 │   └── values/                       # Helm value overrides for infra
-│       ├── argocd-values.yaml
-│       ├── prometheus-values.yaml
-│       ├── grafana-values.yaml
-│       ├── loki-values.yaml
-│       ├── tempo-values.yaml
-│       └── fluent-bit-values.yaml
+│       ├── base/                     # Shared values (all clusters)
+│       │   ├── traefik-values.yaml
+│       │   ├── keycloak-values.yaml
+│       │   ├── grafana-values.yaml
+│       │   ├── prometheus-values.yaml
+│       │   ├── gitea-values.yaml
+│       │   └── ...
+│       ├── upc-dev/                  # upc-dev cluster-specific values
+│       │   ├── traefik-values.yaml
+│       │   ├── keycloak-values.yaml
+│       │   └── grafana-values.yaml
+│       └── upc-prod/                # upc-prod cluster-specific values
+│           ├── traefik-values.yaml
+│           ├── keycloak-values.yaml
+│           └── grafana-values.yaml
 │
-├── apps/                             # Business Application ArgoCD manifests
-│   ├── mcp10x.yaml                   # MCP 10X application
-│   ├── musicman.yaml                 # Music Man application
-│   ├── dot-ai-stack.yaml             # Dot AI Stack
-│   └── argo-mcp.yaml                 # ArgoCD MCP server
+├── apps/                             # Business Application ArgoCD manifests (Kustomize)
+│   ├── base/                         # Base app manifests
+│   │   ├── kustomization.yaml
+│   │   ├── dot-ai-stack.yaml
+│   │   └── ...
+│   └── overlays/
+│       ├── upc-dev/                  # Uses base as-is
+│       └── upc-prod/                # Patches value paths
 │
 ├── cluster-resources/                # Cluster-wide Kubernetes resources
-│   ├── cert-manager-namespace.yaml
-│   ├── secrets-namespace.yaml
-│   ├── letsencrypt-issuer.yaml       # Let's Encrypt ClusterIssuer
-│   ├── kyverno-config.yaml
-│   ├── argocd-notifications-secret-sealed.yaml
-│   ├── forte10x-repo-credentials-sealed.yaml
-│   ├── mcp10x-repo-credentials-sealed.yaml
+│   ├── ...
 │   └── policies/                     # Kyverno policies
-│       ├── deployment-verifier.yaml
-│       ├── label-checker.yaml
-│       ├── bare-pod-cleaner.yaml
-│       ├── replicaset-cleaner.yaml
-│       ├── default-ns-blocker.yaml
-│       ├── secret-cloner.yaml
-│       └── auth-sidecar-injector.yaml
 │
-├── secrets/                          # Application secrets (sealed)
-│   ├── argocd-mcp-credentials.yaml
-│   ├── dot-ai-secrets.yaml
-│   ├── mcp10x-credentials-sealed.yaml
-│   └── musicman-credentials.yaml
+├── secrets/                          # Application secrets (sealed, per-cluster)
+│   └── upc-dev/                      # Secrets for upc-dev cluster
 │
 ├── private/                          # Local-only files (NOT in Git)
-│   ├── *.yaml                        # Unsealed secrets
-│   └── *.sh                          # Helper scripts
 │
 └── docs/                             # Documentation
-    ├── GITOPS-ARCHITECTURE.md        # This file
-    ├── DEVELOPER-GUIDE.md
-    ├── OPERATIONS-RUNBOOK.md
-    └── REFERENCE.md
 ```

 **Key Points**:
- `_app-of-apps.yaml` is the root Application that ArgoCD monitors
- `infra/enterprise-apps.yaml` auto-discovers all apps in `apps/` folder
+- `_app-of-apps-upc-dev.yaml` and `_app-of-apps-upc-prod.yaml` are the per-cluster root Applications
+- Kustomize overlays in `infra/overlays/` render base Applications with per-cluster patches
+- Helm values are split: `values/base/` (shared) + `values/upc-dev/` or `values/upc-prod/` (cluster-specific)
+- `apps/` follows the same base/overlays pattern for business applications
 - Changes pushed to this repo trigger automatic syncs in ArgoCD
 - `private/` folder contains local-only files (Git-ignored)

@@ -295,7 +289,7 @@ app-repository/
 ### The App-of-Apps Pattern

 ```
-_app-of-apps.yaml (Root)
+_app-of-apps-{upc-dev,upc-prod}.yaml (Root, per cluster)
    │
    ├── infrastructure-apps (manages infra/)
    │   ├── cluster-resources-application
@@ -315,10 +309,10 @@ _app-of-apps.yaml (Root)
 ```

 **How It Works**:
-1. Bootstrap script installs ArgoCD and applies `_app-of-apps.yaml`
-2. ArgoCD creates the root Application which monitors `infra/` folder
-3. Each YAML in `infra/` becomes a child Application
-4. `enterprise-apps.yaml` monitors `apps/` folder and auto-discovers applications
+1. Bootstrap script installs ArgoCD and applies `_app-of-apps-upc-dev.yaml` (or `upc-prod`)
+2. ArgoCD creates the root Application which monitors the appropriate `infra/overlays/` folder
+3. Kustomize renders base Applications with cluster-specific patches
+4. `enterprise-apps` Application monitors the cluster's `apps/overlays/` folder
 5. ArgoCD continuously syncs (every 60s) and auto-heals drift

 ### Sync Waves & Ordering
@@ -363,6 +357,34 @@ spec:
 - Easy to update all apps by changing the chart
 - Environment-specific values isolated in separate repo

+### Multi-Cluster Pattern
+
+Kustomize overlays enable deploying the same Applications across clusters with different configurations:
+
+```yaml
+# infra/base/ contains default (upc-dev) Applications
+# Helm values are layered: base + cluster-specific
+valueFiles:
+- $values/infra/values/base/traefik-values.yaml    # Shared config
+- $values/infra/values/upc-dev/traefik-values.yaml  # Cluster-specific
+
+# infra/overlays/upc-prod/kustomization.yaml patches the second valueFile
+patches:
+- target:
+    kind: Application
+    name: traefik
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/traefik-values.yaml
+```
+
+**Benefits**:
+- Single source of truth for Application definitions
+- Cluster-specific values isolated per overlay
+- Easy to add new clusters by creating a new overlay
+- Base values shared across all clusters reduce duplication
+
 ---

 ## CI/CD Pipeline
--- a/docs/OPERATIONS-RUNBOOK.md
+++ b/docs/OPERATIONS-RUNBOOK.md
@@ -207,7 +207,7 @@ kubectl get secrets -n argocd -l argocd.argoproj.io/secret-type=repository
 # Settings → Repositories → Should show "Successful" status

 # Test by creating an application
-kubectl apply -f _app-of-apps.yaml
+kubectl apply -f _app-of-apps-upc-dev.yaml   # or _app-of-apps-upc-prod.yaml

 # Check application sync status
 kubectl get applications -n argocd
@@ -1352,13 +1352,13 @@ kubectl get deployment argocd-server -n argocd \
  -o jsonpath='{.spec.template.spec.containers[0].image}'

 # Update version in values
-vim infra/values/argocd-values.yaml
+vim infra/values/base/argocd-values.yaml

 # Or upgrade via Helm directly
 helm upgrade argocd argo-cd \
  --repo https://argoproj.github.io/argo-helm \
  --namespace argocd \
-  --values infra/values/argocd-values.yaml \
+  --values infra/values/base/argocd-values.yaml \
  --version 6.0.0  # New version

 # Verify
@@ -1454,8 +1454,8 @@ kubectl top pods --all-namespaces --sort-by=cpu
 Example: Adding Redis

 ```bash
-# 1. Create application manifest
-cat > infra/redis-application.yaml <<EOF
+# 1. Create application manifest in base/
+cat > infra/base/redis-application.yaml <<EOF
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
@@ -1465,15 +1465,17 @@ metadata:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: default
-  source:
-    repoURL: https://charts.bitnami.com/bitnami
+  sources:
+  - repoURL: https://charts.bitnami.com/bitnami
    chart: redis
    targetRevision: 18.0.0
    helm:
-      values: |
-        auth:
-          enabled: true
-          password: changeme
+      releaseName: redis
+      valueFiles:
+      - \$values/infra/values/base/redis-values.yaml
+  - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
+    targetRevision: HEAD
+    ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: redis
@@ -1485,30 +1487,37 @@ spec:
    - CreateNamespace=true
 EOF

-# 2. Commit and push
-git add infra/redis-application.yaml
+# 2. Add to base kustomization
+# Edit infra/base/kustomization.yaml and add: - redis-application.yaml
+
+# 3. Create base values file
+cat > infra/values/base/redis-values.yaml <<EOF
+auth:
+  enabled: true
+EOF
+
+# 4. Commit and push
+git add infra/base/redis-application.yaml infra/values/base/redis-values.yaml infra/base/kustomization.yaml
 git commit -m "Add Redis infrastructure component"
 git push

-# 3. ArgoCD will auto-sync within 60 seconds
+# 5. ArgoCD will auto-sync within 60 seconds
 ```

-### Multi-Cluster Setup (Future)
+### Multi-Cluster Setup

-For multi-cluster deployments:
+The repository supports multiple clusters via Kustomize overlays:

-```yaml
-# Different destinations per environment
-# dev-cluster
-destination:
-  server: https://dev.k8s.example.com
-  namespace: myapp
+- **upc-dev** (default): `infra/overlays/upc-dev/` — uses base Applications as-is
+- **upc-prod**: `infra/overlays/upc-prod/` — patches value file paths from `upc-dev` to `upc-prod`

-# prod-cluster
-destination:
-  server: https://prod.k8s.example.com
-  namespace: myapp
-```
+Each cluster has its own:
+- Root app-of-apps file: `_app-of-apps-upc-dev.yaml` / `_app-of-apps-upc-prod.yaml`
+- Cluster-specific Helm values: `infra/values/upc-dev/` / `infra/values/upc-prod/`
+- Sealed secrets: `secrets/upc-dev/` (others as needed)
+- Apps overlay: `apps/overlays/upc-dev/` / `apps/overlays/upc-prod/`
+
+To add a new cluster, create a new overlay directory (e.g., `infra/overlays/upc-staging/`) with patches that swap the value file paths.

 ### Blue-Green Deployments

--- a/docs/README.md
+++ b/docs/README.md
@@ -180,7 +180,7 @@ Reference for:
                            │
                            ▼
 ┌──────────────────────────────────────────────────────────────┐
-│               Kubernetes Cluster (UpCloud)                    │
+│          Kubernetes Clusters (UpCloud: upc-dev, upc-prod)     │
 │  ┌──────────────────────────────────────────────────────┐    │
 │  │  Infrastructure: Traefik, Cert-Manager, Kyverno     │    │
 │  ├──────────────────────────────────────────────────────┤    │
@@ -194,7 +194,7 @@ Reference for:
 ### Key Technologies

 - **GitOps**: ArgoCD
- **Kubernetes**: UpCloud Managed Kubernetes
+- **Kubernetes**: UpCloud Managed Kubernetes (multi-cluster: upc-dev, upc-prod)
 - **Ingress**: Traefik v2
 - **Certificates**: Cert-Manager + Let's Encrypt
 - **Policies**: Kyverno
--- a/docs/REFERENCE.md
+++ b/docs/REFERENCE.md
@@ -21,7 +21,7 @@
 |-----------|-------|
 | **Provider** | UpCloud Managed Kubernetes |
 | **Environment** | Production (internal use) |
-| **Cluster Count** | Single cluster |
+| **Cluster Count** | Multi-cluster (upc-dev, upc-prod) |
 | **GitOps Tool** | ArgoCD |
 | **Ingress Controller** | Traefik v2 |
 | **Certificate Management** | Cert-Manager + Let's Encrypt |
@@ -71,7 +71,8 @@ Internet
 ```
 launchpad/
 ├── bootstrap.sh                   # Cluster initialization script
-├── _app-of-apps.yaml              # Root ArgoCD Application
+├── _app-of-apps-upc-dev.yaml     # Root ArgoCD Application (upc-dev)
+├── _app-of-apps-upc-prod.yaml    # Root ArgoCD Application (upc-prod)
 │
 ├── infra/                         # Infrastructure applications
 │   ├── cluster-resources-application.yaml
@@ -156,15 +157,15 @@ ArgoCd() {
  helm upgrade --install argocd argo-cd \
    --repo https://argoproj.github.io/argo-helm \
    --namespace argocd --create-namespace \
-    --values infra/values/argocd-values.yaml \
+    --values infra/values/base/argocd-values.yaml \
    --set notifications.context.clusterName="$CLUSTER_NAME" \
    --timeout 60s --atomic

-  kubectl apply -f _app-of-apps.yaml -n argocd
+  kubectl apply -f _app-of-apps-upc-dev.yaml -n argocd   # or _app-of-apps-upc-prod.yaml
 }
 ```

-**`_app-of-apps.yaml`**
+**`_app-of-apps-upc-dev.yaml`** / **`_app-of-apps-upc-prod.yaml`**
 ```yaml
 apiVersion: argoproj.io/v1alpha1
 kind: Application
@@ -615,7 +616,7 @@ retry:

 **Configuration**:
 ```yaml
-# infra/traefik-application.yaml
+# infra/base/traefik-application.yaml
 replicas: 2

 service:
@@ -790,7 +791,7 @@ persistence:

 **Configuration**:
 ```yaml
-# infra/gitea.yaml + infra/values/gitea-values.yaml
+# infra/base/gitea.yaml + infra/values/base/gitea-values.yaml
 ingress:
  host: git.forteapps.net
  tls: cert-manager (letsencrypt-prod)
@@ -833,7 +834,7 @@ postgresql:

 **Configuration**:
 ```yaml
-# infra/gitea-actions.yaml + infra/values/gitea-actions-values.yaml
+# infra/base/gitea-actions.yaml + infra/values/base/gitea-actions-values.yaml
 replicaCount: 3

 runner:
@@ -994,7 +995,7 @@ kubectl get secret keycloak-client-<app> -n keycloak -o jsonpath='{.metadata.ann

 **Configuration**:
 ```yaml
-# infra/renovate.yaml + infra/values/renovate-values.yaml
+# infra/base/renovate.yaml + infra/values/base/renovate-values.yaml
 cronjob:
  schedule: "@daily"
  concurrencyPolicy: Forbid