clusters

Reviewed-on: #3
Reviewed-by: gitea_admin <admin@forteapps.net>
Co-authored-by: Danijel Simeunovic <danijel.simeunovic@fortedigital.com>
Co-committed-by: Danijel Simeunovic <danijel.simeunovic@fortedigital.com>

README.md

@@ -83,20 +83,26 @@ This repository contains the complete GitOps configuration for our Kubernetes cl
 ├── bootstrap.sh                # Cluster initialization script
 ├── _app-of-apps.yaml          # Root ArgoCD Application (App-of-Apps pattern)
 │
-├── infra/                     # Infrastructure ArgoCD Applications
+├── infra/                     # Infrastructure ArgoCD Applications (Kustomize multi-cluster)
-│   ├── enterprise-apps.yaml   # Manages all apps in apps/ folder
+│   ├── base/                  # Base ArgoCD Application manifests (EU defaults)
-│   ├── traefik-application.yaml
+│   │   ├── kustomization.yaml
-│   ├── cert-manager-application.yaml
+│   │   ├── traefik-application.yaml
-│   ├── kyverno.yaml
+│   │   ├── keycloak.yaml
-│   ├── prometheus.yaml
+│   │   ├── grafana.yaml
-│   ├── grafana.yaml
+│   │   ├── gitea.yaml
-│   ├── loki.yaml
+│   │   ├── gitea-actions.yaml
-│   ├── tempo.yaml
+│   │   ├── tempo.yaml
-│   ├── fluent-bit.yaml
+│   │   ├── renovate.yaml
-│   ├── trivy.yaml
+│   │   ├── ...                # All other Application manifests
-│   ├── sealedsecrets.yaml
+│   │   └── secrets.yaml
-│   ├── renovate.yaml
+│   ├── overlays/              # Per-cluster overrides
+│   │   ├── upc-dev/           # UpCloud Dev cluster (uses base as-is)
+│   │   └── upc-prod/         # UpCloud Prod cluster (patches value paths)
+│   ├── dashboards/            # Grafana dashboard ConfigMaps
 │   └── values/                # Helm value overrides
+│       ├── base/              # Shared values (all clusters)
+│       ├── upc-dev/           # UpCloud Dev-specific values
+│       └── upc-prod/         # UpCloud Prod-specific values
 │
 ├── apps/                      # Business Applications
 │   ├── mcp10x.yaml
@@ -355,7 +361,7 @@ kubectl patch application myapp -n argocd \
 ## 📖 Key Concepts
 
 ### App-of-Apps Pattern
-`_app-of-apps.yaml` is the root Application that manages all other Applications in `infra/`. Each YAML in `infra/` becomes a child Application managed by ArgoCD.
+`_app-of-apps.yaml` is the root Application that manages all other Applications in `infra/`. Kustomize overlays in `infra/overlays/{upc-dev,upc-prod}/` render the base Applications with per-cluster patches (e.g., swapping value file paths from `upc-dev` to `upc-prod`).
 
 ### Multi-Source Pattern
 Applications reference both:
@@ -454,14 +460,14 @@ Documentation lives in `docs/`. To update:
 ### Current Environment
 - **Provider**: UpCloud Managed Kubernetes
 - **Environment**: Production (internal use only)
-- **Cluster**: Single cluster
+- **Clusters**: Multi-cluster (upc-dev, upc-prod) via Kustomize overlays
 - **Auth**: Disabled for ArgoCD (internal access)
 - **Backup**: None (cluster rebuildable via GitOps)
 
 ### Known Limitations
 - No automated backups (yet)
 - Secret rotation not automated
-- Single cluster (no multi-cluster setup)
+- Multi-cluster limited to upc-dev and upc-prod environments
 - DNS management is manual
 
 **Future improvements**: See [Operations Runbook - Disaster Recovery](docs/OPERATIONS-RUNBOOK.md#disaster-recovery)

_app-of-apps-upc-dev.yaml

@@ -20,7 +20,7 @@ spec:
   source:
     repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD
-    path: infra
+    path: infra/overlays/upc-dev
   destination:
     server: https://kubernetes.default.svc
     namespace: default

_app-of-apps-upc-prod.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: infrastructure-apps
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: infrastructure-apps
+    app.kubernetes.io/part-of: platform
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: git@github.com:fortedigital/sturdy-adventure.git
+    targetRevision: HEAD
+    path: infra/overlays/upc-prod
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: default
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true

apps/base/dot-ai-stack.yaml

@@ -27,29 +27,19 @@ metadata:
 spec:
   project: default
 
-  source:
+  sources:
-    repoURL: ghcr.io/vfarcic/dot-ai-stack/charts
+  - repoURL: ghcr.io/vfarcic/dot-ai-stack/charts
     chart: dot-ai-stack
     targetRevision: "0.56.0"
-
     helm:
       releaseName: dot-ai-stack
-      values: |
+      valueFiles:
-        dot-ai:
+      - $values/infra/values/base/dot-ai-stack-values.yaml
-          ingress:
+      - $values/infra/values/upc-dev/dot-ai-stack-values.yaml
-            enabled: true
+
-            className: traefik
+  - repoURL: git@github.com:fortedigital/sturdy-adventure.git
-            host: kubemcp.forteapps.net
+    targetRevision: HEAD
-          webUI:
+    ref: values
-            baseUrl: http://kubemcpui.forteapps.net
-        dot-ai-ui:
-          uiAuth:
-            secretRef:
-              name: dot-ai-secrets
-          ingress:
-            enabled: true
-            className: traefik
-            host: kubemcpui.forteapps.net
 
   destination:
     server: https://kubernetes.default.svc

apps/base/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- dot-ai-stack.yaml
+- mcp10x.yaml
+- musicman.yaml
+- mcpcoder.yaml
+- argo-mcp.yaml

apps/overlays/upc-dev/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+
+# No patches needed — base already has "upc-dev" paths
+# upc-dev is the default/base cluster

apps/overlays/upc-prod/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+
+patches:
+# dot-ai-stack: swap upc-dev → upc-prod
+- target:
+    kind: Application
+    name: dot-ai-stack
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/dot-ai-stack-values.yaml

bootstrap.sh

@@ -2,7 +2,14 @@
 # in case of  $'\r': command not found error, run command below first
 # sed -i 's/\r$//' ./bootstrap.sh
 
-echo "running $0..."
+CLUSTER="${1:?Usage: ./bootstrap.sh <cluster> (eu|us)}"
+
+echo "running $0 for cluster: ${CLUSTER}..."
+
+# Source cluster config
+eval $(yq -r 'to_entries[] | "export \(.key)=\"\(.value)\""' "clusters/${CLUSTER}.yaml")
+
+echo "Bootstrapping cluster: ${clusterName} (${CLUSTER})..."
 
 ############################################################
 # Bootstrap                                                     #
@@ -20,8 +27,8 @@ Bootstrap()
 Github()
 {
     echo "Installing secret..."
-    kubectl apply -f private/github.yaml
+    kubectl apply -f private/github-${CLUSTER}.yaml
-    kubectl apply -f private/main.key
+    kubectl apply -f private/main-${CLUSTER}.key
 }
 
 ############################################################
@@ -31,15 +38,15 @@ ArgoCd()
 {
     # install argocd
     echo "Installing ArgoCD..."
-    CLUSTER_NAME="${CLUSTER_NAME:-dev-fd-no-svg1}"
     helm upgrade --install argocd argo-cd \
             --repo https://argoproj.github.io/argo-helm \
             --namespace argocd --create-namespace \
-            --values infra/values/argocd-values.yaml \
+            --values infra/values/base/argocd-values.yaml \
-            --set notifications.context.clusterName="$CLUSTER_NAME" \
+            --values "infra/values/${CLUSTER}/argocd-values.yaml" \
+            --set notifications.context.clusterName="${clusterName}" \
             --timeout 60s --atomic
 
-    kubectl apply -f _app-of-apps.yaml -n argocd
+    kubectl apply -f "_app-of-apps-${CLUSTER}.yaml" -n argocd
 }
 
-Bootstrap
+# Bootstrap

cluster-resources/policies/auth-sidecar-injector.yaml

@@ -243,8 +243,8 @@ spec:
             - name: AUTH_OIDC_CLIENT_SECRET
               valueFrom:
                 secretKeyRef:
-                  name: auth-oidc
+                  name: "{{ request.object.metadata.annotations.\"policies.forteapps.io/auth-oidc-credentials-secret\" || 'auth-oidc' }}"
-                  key: client-secret
+                  key: "{{ request.object.metadata.annotations.\"policies.forteapps.io/auth-oidc-credentials-secret-key\" || 'client-secret' }}"
             resources:
               limits:
                 cpu: 50m
@@ -410,8 +410,8 @@ spec:
             - name: AUTH_OAUTH_CLIENT_SECRET
               valueFrom:
                 secretKeyRef:
-                  name: auth-oauth
+                  name: "{{ request.object.metadata.annotations.\"policies.forteapps.io/auth-oauth-credentials-secret\" || 'auth-oauth' }}"
-                  key: client-secret
+                  key: "{{ request.object.metadata.annotations.\"policies.forteapps.io/auth-oauth-credentials-secret-key\" || 'client-secret' }}"
             - name: AUTH_OAUTH_DELEGATION_CLIENT_SECRET
               valueFrom:
                 secretKeyRef:

cluster-resources/policies/keycloak-client-cloner.yaml

@@ -0,0 +1,37 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: keycloak-client-config-cloner
+spec:
+  rules:
+  - name: clone-client-config-to-keycloak
+    skipBackgroundRequests: false
+    match:
+      any:
+      - resources:
+          kinds:
+          - Secret
+          selector:
+            matchLabels:
+              keycloak.forteapps.net/client-config: "true"
+    exclude:
+      any:
+      - resources:
+          namespaces:
+          - keycloak
+    generate:
+      apiVersion: v1
+      kind: Secret
+      name: "{{request.object.metadata.name}}"
+      namespace: keycloak
+      synchronize: true
+      data:
+        metadata:
+          labels:
+            keycloak.forteapps.net/client-config: "true"
+            keycloak.forteapps.net/source-namespace: "{{request.object.metadata.namespace}}"
+          annotations:
+            keycloak.forteapps.net/source-name: "{{request.object.metadata.name}}"
+            keycloak.forteapps.net/source-namespace: "{{request.object.metadata.namespace}}"
+        data: "{{request.object.data}}"
+        type: "{{request.object.type}}"

clusters/upc-dev.yaml

@@ -0,0 +1,10 @@
+clusterName: dev-fd-no-svg1
+domain: forteapps.net
+argocdDomain: argocd.127.0.0.1.nip.io
+grafanaDomain: grafana.forteapps.net
+keycloakDomain: id.forteapps.net
+dotaiDomain: kubemcp.forteapps.net
+dotaiUiDomain: kubemcpui.forteapps.net
+letsencryptEmail: danijels@gmail.com
+trustedIPs: "172.16.1.0/24"
+cloudProvider: upcloud

clusters/upc-prod.yaml

@@ -0,0 +1,10 @@
+clusterName: dev-fd-us-east1
+domain: us.forteapps.net
+argocdDomain: argocd.us.forteapps.net
+grafanaDomain: grafana.us.forteapps.net
+keycloakDomain: id.us.forteapps.net
+dotaiDomain: kubemcp.us.forteapps.net
+dotaiUiDomain: kubemcpui.us.forteapps.net
+letsencryptEmail: danijels@gmail.com
+trustedIPs: "10.0.0.0/16"
+cloudProvider: tbd

docs/DEVELOPER-GUIDE.md

@@ -1250,22 +1250,121 @@ kubectl logs -n myapp <pod-name> -c authn
 
 ## Adding a New Keycloak Client
 
-When you need an application to authenticate via Keycloak (OIDC), you can add a client definition to the realm config. The secret syncer automatically extracts the Keycloak-generated client secret into a Kubernetes Secret that your application can reference — no manual secret management needed.
+There are two ways to add an OIDC client, depending on your use case:
 
-### How It Works
+| Method | Best for | Who edits the infra repo? |
+|--------|----------|--------------------------|
+| **Self-service** (recommended) | New apps that deploy their own resources | App developer — no infra changes needed |
+| **Legacy (realm JSON)** | Existing clients already defined in forte-realm.json (e.g., Gitea) | Platform engineer |
 
-1. You define a client in `forte-realm.json` (inside `keycloak-values.yaml`) **without** a `secret` field
+Both methods are served by the **Keycloak Client Registrar** CronJob, which runs every 2 minutes.
-2. Keycloak auto-generates a cryptographically strong secret on first creation
-3. An ArgoCD **PostSync Job** (`keycloak-secret-syncer`) runs after each Keycloak sync:
-   - Authenticates to the Keycloak Admin API
-   - Finds clients with `k8s.secret.sync: "true"` in their attributes
-   - Extracts the auto-generated secret for each client
-   - Creates/updates a K8s Secret in the target namespace with `client-id` and `client-secret` keys
-4. Your application references the syncer-created Secret
 
-### Step 1: Add Client to Realm Config
+### Self-Service OIDC Client Registration
 
-In `infra/values/keycloak-values.yaml`, add a new entry to the `clients` array in `forte-realm.json`:
+This is the recommended flow for new applications. Your app deploys a labeled config Secret in its own namespace; the platform handles everything else.
+
+#### How It Works
+
+1. You deploy a Secret with label `keycloak.forteapps.net/client-config: "true"` containing a `client.json` definition
+2. A **Kyverno ClusterPolicy** (`keycloak-client-config-cloner`) clones it to the `keycloak` namespace
+3. The **Client Registrar CronJob** picks it up within 2 minutes:
+   - Registers (or updates) the client in Keycloak
+   - Fetches the auto-generated client secret
+   - Creates a credential Secret in your app's namespace
+   - Annotates the config Secret with sync status
+
+#### Step 1: Create the Config Secret
+
+Deploy this Secret in your application's namespace (e.g., as part of your Helm chart or Kustomize overlay):
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: keycloak-client-myapp
+  namespace: myapp
+  labels:
+    keycloak.forteapps.net/client-config: "true"
+stringData:
+  client.json: |
+    {
+      "clientId": "myapp",
+      "name": "My Application",
+      "redirectUris": ["https://myapp.forteapps.net/*"],
+      "webOrigins": ["https://myapp.forteapps.net"],
+      "defaultClientScopes": ["openid", "email", "profile"],
+      "protocolMappers": [],
+      "secret": {
+        "namespace": "myapp",
+        "name": "myapp-oidc-credentials",
+        "keys": { "clientId": "client-id", "clientSecret": "client-secret" }
+      }
+    }
+```
+
+**`client.json` fields**:
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `clientId` | Yes | Keycloak client ID |
+| `name` | Yes | Display name in Keycloak |
+| `redirectUris` | Yes | Allowed redirect URIs |
+| `webOrigins` | Yes | Allowed web origins (CORS) |
+| `defaultClientScopes` | No | Scopes (default: `["openid", "email", "profile"]`) |
+| `protocolMappers` | No | Custom claim mappers (default: `[]`) |
+| `secret.namespace` | No | Namespace for the credential Secret (default: source namespace) |
+| `secret.name` | No | Name of the credential Secret (default: `<clientId>-oidc-credentials`) |
+| `secret.keys.clientId` | No | Key name for client ID in credential Secret (default: `client-id`) |
+| `secret.keys.clientSecret` | No | Key name for client secret in credential Secret (default: `client-secret`) |
+
+#### Step 2: Reference the Credential Secret
+
+In your application's deployment config, reference the credential Secret that the registrar creates:
+
+```yaml
+env:
+- name: OIDC_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: myapp-oidc-credentials
+      key: client-id
+- name: OIDC_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: myapp-oidc-credentials
+      key: client-secret
+```
+
+#### Step 3: Deploy and Wait
+
+Commit and push your changes. The credential Secret will appear within 2 minutes:
+
+```bash
+# Watch for the credential Secret to be created
+kubectl get secret myapp-oidc-credentials -n myapp -w
+
+# Check registrar logs
+kubectl logs -n keycloak job/$(kubectl get jobs -n keycloak --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}')
+
+# Check sync status on the config Secret
+kubectl get secret keycloak-client-myapp -n keycloak -o jsonpath='{.metadata.annotations}'
+```
+
+#### Change Detection
+
+The registrar computes a SHA-256 hash of `client.json` and stores it as an annotation. On subsequent runs, it skips processing if:
+- The hash hasn't changed, AND
+- The credential Secret already exists in the target namespace
+
+To force a re-sync, update any field in `client.json` (e.g., add a trailing space to `name`).
+
+### Legacy Method: Realm JSON
+
+Existing clients (like Gitea) are defined directly in `forte-realm.json` inside `keycloak-values.yaml`. The registrar syncs their secrets via client attributes.
+
+#### Step 1: Add Client to Realm Config
+
+In `infra/values/base/keycloak-values.yaml`, add a new entry to the `clients` array in `forte-realm.json`:
 
 ```json
 {
@@ -1292,59 +1391,27 @@ In `infra/values/keycloak-values.yaml`, add a new entry to the `clients` array i
 
 **Important**:
 - Do **NOT** include a `"secret"` field — Keycloak generates one automatically
-- The `attributes` block tells the syncer where to create the K8s Secret
+- The `attributes` block tells the registrar where to create the K8s Secret
-- The target namespace must exist before the syncer runs (ArgoCD creates it via `CreateNamespace=true`)
 - Set `client-id-key` / `client-secret-key` to match what the consuming app expects (defaults: `client-id` / `client-secret`)
 
-### Step 2: Reference the Secret in Your Application
+#### Step 2: Reference the Secret in Your Application
-
-In your application's Helm values, reference the syncer-created secret:
 
 ```yaml
-# In helm-values/myapp/values.yaml (or inline in values file)
-# The secret will have keys: client-id, client-secret
 existingSecret: myapp-oidc-credentials
 ```
 
-For Gitea-style oauth config:
+#### Step 3: Commit and Push
-```yaml
-oauth:
-- name: "Forte"
-  provider: "openidConnect"
-  existingSecret: myapp-oidc-credentials # Gitea expects "key" and "secret" as fields
-  autoDiscoverUrl: "https://id.forteapps.net/realms/forte/.well-known/openid-configuration"
-```
-
-### Step 3: Commit and Push
 
 ```bash
 cd ~/dev/k8s/launchpad
-git add infra/values/keycloak-values.yaml
+git add infra/values/base/keycloak-values.yaml
 git commit -m "Add myapp Keycloak client with auto-sync"
 git push
 ```
 
-ArgoCD will:
+ArgoCD will sync the Keycloak config, and the registrar CronJob will pick up the new client within 2 minutes.
-1. Sync the Keycloak config (keycloakConfigCli creates the client)
-2. Run the PostSync syncer Job
-3. The syncer creates `myapp-oidc-credentials` in the `myapp` namespace
 
-### Step 4: Verify
+#### Legacy Sync Attribute Reference
-
-```bash
-# Check the syncer job ran successfully
-kubectl get jobs -n keycloak
-kubectl logs -n keycloak job/keycloak-secret-syncer
-
-# Verify the secret was created
-kubectl get secret myapp-oidc-credentials -n myapp -o yaml
-
-# Check the secret has the expected keys
-kubectl get secret myapp-oidc-credentials -n myapp -o jsonpath='{.data.client-id}' | base64 -d
-kubectl get secret myapp-oidc-credentials -n myapp -o jsonpath='{.data.client-secret}' | base64 -d
-```
-
-### Sync Attribute Reference
 
 | Attribute | Required | Default | Description |
 |-----------|----------|---------|-------------|
@@ -1354,11 +1421,9 @@ kubectl get secret myapp-oidc-credentials -n myapp -o jsonpath='{.data.client-se
 | `k8s.secret.client-id-key` | No | `client-id` | Field name for the client ID in the K8s Secret |
 | `k8s.secret.client-secret-key` | No | `client-secret` | Field name for the client secret in the K8s Secret |
 
-**Note on key names:** Different applications expect different field names. For example, the Gitea Helm chart expects `key` and `secret`, while a generic OIDC consumer might expect `client-id` and `client-secret`. Use the optional key attributes to match what the consuming application expects.
-
 ### Retrieving Secrets for External Deployments
 
-The syncer always writes a **central copy** of every synced secret to the `secrets` namespace, in addition to the target namespace. This allows operators to retrieve client credentials for applications deployed outside this cluster:
+The registrar always writes a **central copy** of every synced secret to the `secrets` namespace, in addition to the target namespace. This allows operators to retrieve client credentials for applications deployed outside this cluster:
 
 ```bash
 # View the central copy
@@ -1369,16 +1434,13 @@ kubectl get secret myapp-oidc-credentials -n secrets \
   -o jsonpath='{.data.client-secret}' | base64 -d
 ```
 
-This is useful when an application runs on a separate cluster or external infrastructure and needs the Keycloak-generated OIDC credentials provisioned manually (e.g., via a SealedSecret on the remote side).
+### Registrar Behavior Notes
 
-### Syncer Behavior Notes
+- The registrar runs as a CronJob every 2 minutes (`concurrencyPolicy: Forbid`)
-
-- The syncer runs as an ArgoCD **PostSync hook** — it executes after all Keycloak resources are healthy
-- `BeforeHookCreation` delete policy ensures old Job is cleaned up before each run
 - If the target namespace doesn't exist, the target write is skipped with a warning (the central copy still happens)
 - A central copy is **always** written to the `secrets` namespace for every synced client
-- The syncer uses the `keycloak-credentials` secret for admin authentication
+- The registrar uses the `keycloak-credentials` secret for admin authentication
-- Created secrets have the label `app.kubernetes.io/managed-by: keycloak-secret-syncer`
+- Created secrets have the label `app.kubernetes.io/managed-by: keycloak-client-registrar`
 
 ---
 

docs/GITOPS-ARCHITECTURE.md

@@ -16,7 +16,7 @@ This Kubernetes cluster uses a **GitOps approach** powered by **ArgoCD**, where
 
 ### Key Characteristics
 - **Environment**: Production (internal use only)
-- **Cluster Type**: Single cluster, single environment
+- **Cluster Type**: Multi-cluster (upc-dev, upc-prod) via Kustomize overlays
 - **GitOps Tool**: ArgoCD
 - **Deployment Pattern**: App-of-Apps
 - **Secret Management**: Sealed Secrets (kubeseal)
@@ -62,8 +62,8 @@ This Kubernetes cluster uses a **GitOps approach** powered by **ArgoCD**, where
                                                         │
                                                         ▼
                                            ┌────────────────────────────────┐
-                                           │   Kubernetes Cluster           │
+                                           │   Kubernetes Clusters          │
-                                           │   (UpCloud Managed)            │
+                                           │   (UpCloud: upc-dev, upc-prod) │
                                            │                                │
                                            │  ┌──────────────────────────┐  │
                                            │  │    ArgoCD                │  │
@@ -116,74 +116,68 @@ This Kubernetes cluster uses a **GitOps approach** powered by **ArgoCD**, where
 ```
 launchpad/
 ├── bootstrap.sh                      # Cluster initialization script
-├── _app-of-apps.yaml                 # Root ArgoCD Application (App-of-Apps pattern)
+├── _app-of-apps-upc-dev.yaml        # Root ArgoCD Application (upc-dev cluster)
+├── _app-of-apps-upc-prod.yaml       # Root ArgoCD Application (upc-prod cluster)
 │
-├── infra/                            # Infrastructure ArgoCD Applications
+├── infra/                            # Infrastructure ArgoCD Applications (Kustomize)
-│   ├── enterprise-apps.yaml          # Parent app managing all apps in apps/
+│   ├── base/                         # Base Application manifests (upc-dev defaults)
-│   ├── cluster-resources-application.yaml
+│   │   ├── kustomization.yaml
-│   ├── traefik-application.yaml
+│   │   ├── traefik-application.yaml
-│   ├── cert-manager-application.yaml
+│   │   ├── keycloak.yaml
-│   ├── kyverno.yaml
+│   │   ├── grafana.yaml
-│   ├── kyverno-policies.yaml
+│   │   ├── gitea.yaml
-│   ├── prometheus.yaml
+│   │   ├── gitea-actions.yaml
-│   ├── grafana.yaml
+│   │   ├── tempo.yaml
-│   ├── loki.yaml
+│   │   ├── renovate.yaml
-│   ├── tempo.yaml
+│   │   ├── ...                       # All other Application manifests
-│   ├── fluent-bit.yaml
+│   │   └── secrets.yaml
-│   ├── trivy.yaml
+│   ├── overlays/                     # Per-cluster overrides
-│   ├── sealedsecrets.yaml
+│   │   ├── upc-dev/                  # UpCloud Dev (uses base as-is)
-│   ├── secrets.yaml
+│   │   └── upc-prod/                # UpCloud Prod (patches value paths)
+│   ├── dashboards/                   # Grafana dashboard ConfigMaps
 │   └── values/                       # Helm value overrides for infra
-│       ├── argocd-values.yaml
+│       ├── base/                     # Shared values (all clusters)
-│       ├── prometheus-values.yaml
+│       │   ├── traefik-values.yaml
-│       ├── grafana-values.yaml
+│       │   ├── keycloak-values.yaml
-│       ├── loki-values.yaml
+│       │   ├── grafana-values.yaml
-│       ├── tempo-values.yaml
+│       │   ├── prometheus-values.yaml
-│       └── fluent-bit-values.yaml
+│       │   ├── gitea-values.yaml
+│       │   └── ...
+│       ├── upc-dev/                  # upc-dev cluster-specific values
+│       │   ├── traefik-values.yaml
+│       │   ├── keycloak-values.yaml
+│       │   └── grafana-values.yaml
+│       └── upc-prod/                # upc-prod cluster-specific values
+│           ├── traefik-values.yaml
+│           ├── keycloak-values.yaml
+│           └── grafana-values.yaml
 │
-├── apps/                             # Business Application ArgoCD manifests
+├── apps/                             # Business Application ArgoCD manifests (Kustomize)
-│   ├── mcp10x.yaml                   # MCP 10X application
+│   ├── base/                         # Base app manifests
-│   ├── musicman.yaml                 # Music Man application
+│   │   ├── kustomization.yaml
-│   ├── dot-ai-stack.yaml             # Dot AI Stack
+│   │   ├── dot-ai-stack.yaml
-│   └── argo-mcp.yaml                 # ArgoCD MCP server
+│   │   └── ...
+│   └── overlays/
+│       ├── upc-dev/                  # Uses base as-is
+│       └── upc-prod/                # Patches value paths
 │
 ├── cluster-resources/                # Cluster-wide Kubernetes resources
-│   ├── cert-manager-namespace.yaml
+│   ├── ...
-│   ├── secrets-namespace.yaml
-│   ├── letsencrypt-issuer.yaml       # Let's Encrypt ClusterIssuer
-│   ├── kyverno-config.yaml
-│   ├── argocd-notifications-secret-sealed.yaml
-│   ├── forte10x-repo-credentials-sealed.yaml
-│   ├── mcp10x-repo-credentials-sealed.yaml
 │   └── policies/                     # Kyverno policies
-│       ├── deployment-verifier.yaml
-│       ├── label-checker.yaml
-│       ├── bare-pod-cleaner.yaml
-│       ├── replicaset-cleaner.yaml
-│       ├── default-ns-blocker.yaml
-│       ├── secret-cloner.yaml
-│       └── auth-sidecar-injector.yaml
 │
-├── secrets/                          # Application secrets (sealed)
+├── secrets/                          # Application secrets (sealed, per-cluster)
-│   ├── argocd-mcp-credentials.yaml
+│   └── upc-dev/                      # Secrets for upc-dev cluster
-│   ├── dot-ai-secrets.yaml
-│   ├── mcp10x-credentials-sealed.yaml
-│   └── musicman-credentials.yaml
 │
 ├── private/                          # Local-only files (NOT in Git)
-│   ├── *.yaml                        # Unsealed secrets
-│   └── *.sh                          # Helper scripts
 │
 └── docs/                             # Documentation
-    ├── GITOPS-ARCHITECTURE.md        # This file
-    ├── DEVELOPER-GUIDE.md
-    ├── OPERATIONS-RUNBOOK.md
-    └── REFERENCE.md
 ```
 
 **Key Points**:
-- `_app-of-apps.yaml` is the root Application that ArgoCD monitors
+- `_app-of-apps-upc-dev.yaml` and `_app-of-apps-upc-prod.yaml` are the per-cluster root Applications
-- `infra/enterprise-apps.yaml` auto-discovers all apps in `apps/` folder
+- Kustomize overlays in `infra/overlays/` render base Applications with per-cluster patches
+- Helm values are split: `values/base/` (shared) + `values/upc-dev/` or `values/upc-prod/` (cluster-specific)
+- `apps/` follows the same base/overlays pattern for business applications
 - Changes pushed to this repo trigger automatic syncs in ArgoCD
 - `private/` folder contains local-only files (Git-ignored)
 
@@ -295,7 +289,7 @@ app-repository/
 ### The App-of-Apps Pattern
 
 ```
-_app-of-apps.yaml (Root)
+_app-of-apps-{upc-dev,upc-prod}.yaml (Root, per cluster)
     │
     ├── infrastructure-apps (manages infra/)
     │   ├── cluster-resources-application
@@ -315,10 +309,10 @@ _app-of-apps.yaml (Root)
 ```
 
 **How It Works**:
-1. Bootstrap script installs ArgoCD and applies `_app-of-apps.yaml`
+1. Bootstrap script installs ArgoCD and applies `_app-of-apps-upc-dev.yaml` (or `upc-prod`)
-2. ArgoCD creates the root Application which monitors `infra/` folder
+2. ArgoCD creates the root Application which monitors the appropriate `infra/overlays/` folder
-3. Each YAML in `infra/` becomes a child Application
+3. Kustomize renders base Applications with cluster-specific patches
-4. `enterprise-apps.yaml` monitors `apps/` folder and auto-discovers applications
+4. `enterprise-apps` Application monitors the cluster's `apps/overlays/` folder
 5. ArgoCD continuously syncs (every 60s) and auto-heals drift
 
 ### Sync Waves & Ordering
@@ -363,6 +357,34 @@ spec:
 - Easy to update all apps by changing the chart
 - Environment-specific values isolated in separate repo
 
+### Multi-Cluster Pattern
+
+Kustomize overlays enable deploying the same Applications across clusters with different configurations:
+
+```yaml
+# infra/base/ contains default (upc-dev) Applications
+# Helm values are layered: base + cluster-specific
+valueFiles:
+- $values/infra/values/base/traefik-values.yaml    # Shared config
+- $values/infra/values/upc-dev/traefik-values.yaml  # Cluster-specific
+
+# infra/overlays/upc-prod/kustomization.yaml patches the second valueFile
+patches:
+- target:
+    kind: Application
+    name: traefik
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/traefik-values.yaml
+```
+
+**Benefits**:
+- Single source of truth for Application definitions
+- Cluster-specific values isolated per overlay
+- Easy to add new clusters by creating a new overlay
+- Base values shared across all clusters reduce duplication
+
 ---
 
 ## CI/CD Pipeline

docs/OPERATIONS-RUNBOOK.md

@@ -207,7 +207,7 @@ kubectl get secrets -n argocd -l argocd.argoproj.io/secret-type=repository
 # Settings → Repositories → Should show "Successful" status
 
 # Test by creating an application
-kubectl apply -f _app-of-apps.yaml
+kubectl apply -f _app-of-apps-upc-dev.yaml   # or _app-of-apps-upc-prod.yaml
 
 # Check application sync status
 kubectl get applications -n argocd
@@ -1352,13 +1352,13 @@ kubectl get deployment argocd-server -n argocd \
   -o jsonpath='{.spec.template.spec.containers[0].image}'
 
 # Update version in values
-vim infra/values/argocd-values.yaml
+vim infra/values/base/argocd-values.yaml
 
 # Or upgrade via Helm directly
 helm upgrade argocd argo-cd \
   --repo https://argoproj.github.io/argo-helm \
   --namespace argocd \
-  --values infra/values/argocd-values.yaml \
+  --values infra/values/base/argocd-values.yaml \
   --version 6.0.0  # New version
 
 # Verify
@@ -1454,8 +1454,8 @@ kubectl top pods --all-namespaces --sort-by=cpu
 Example: Adding Redis
 
 ```bash
-# 1. Create application manifest
+# 1. Create application manifest in base/
-cat > infra/redis-application.yaml <<EOF
+cat > infra/base/redis-application.yaml <<EOF
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
@@ -1465,15 +1465,17 @@ metadata:
     argocd.argoproj.io/sync-wave: "1"
 spec:
   project: default
-  source:
+  sources:
-    repoURL: https://charts.bitnami.com/bitnami
+  - repoURL: https://charts.bitnami.com/bitnami
     chart: redis
     targetRevision: 18.0.0
     helm:
-      values: |
+      releaseName: redis
-        auth:
+      valueFiles:
-          enabled: true
+      - \$values/infra/values/base/redis-values.yaml
-          password: changeme
+  - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
+    targetRevision: HEAD
+    ref: values
   destination:
     server: https://kubernetes.default.svc
     namespace: redis
@@ -1485,30 +1487,37 @@ spec:
     - CreateNamespace=true
 EOF
 
-# 2. Commit and push
+# 2. Add to base kustomization
-git add infra/redis-application.yaml
+# Edit infra/base/kustomization.yaml and add: - redis-application.yaml
+
+# 3. Create base values file
+cat > infra/values/base/redis-values.yaml <<EOF
+auth:
+  enabled: true
+EOF
+
+# 4. Commit and push
+git add infra/base/redis-application.yaml infra/values/base/redis-values.yaml infra/base/kustomization.yaml
 git commit -m "Add Redis infrastructure component"
 git push
 
-# 3. ArgoCD will auto-sync within 60 seconds
+# 5. ArgoCD will auto-sync within 60 seconds
 ```
 
-### Multi-Cluster Setup (Future)
+### Multi-Cluster Setup
 
-For multi-cluster deployments:
+The repository supports multiple clusters via Kustomize overlays:
 
-```yaml
+- **upc-dev** (default): `infra/overlays/upc-dev/` — uses base Applications as-is
-# Different destinations per environment
+- **upc-prod**: `infra/overlays/upc-prod/` — patches value file paths from `upc-dev` to `upc-prod`
-# dev-cluster
-destination:
-  server: https://dev.k8s.example.com
-  namespace: myapp
 
-# prod-cluster
+Each cluster has its own:
-destination:
+- Root app-of-apps file: `_app-of-apps-upc-dev.yaml` / `_app-of-apps-upc-prod.yaml`
-  server: https://prod.k8s.example.com
+- Cluster-specific Helm values: `infra/values/upc-dev/` / `infra/values/upc-prod/`
-  namespace: myapp
+- Sealed secrets: `secrets/upc-dev/` (others as needed)
-```
+- Apps overlay: `apps/overlays/upc-dev/` / `apps/overlays/upc-prod/`
+
+To add a new cluster, create a new overlay directory (e.g., `infra/overlays/upc-staging/`) with patches that swap the value file paths.
 
 ### Blue-Green Deployments
 

docs/README.md

@@ -180,7 +180,7 @@ Reference for:
                             │
                             ▼
 ┌──────────────────────────────────────────────────────────────┐
-│               Kubernetes Cluster (UpCloud)                    │
+│          Kubernetes Clusters (UpCloud: upc-dev, upc-prod)     │
 │  ┌──────────────────────────────────────────────────────┐    │
 │  │  Infrastructure: Traefik, Cert-Manager, Kyverno     │    │
 │  ├──────────────────────────────────────────────────────┤    │
@@ -194,7 +194,7 @@ Reference for:
 ### Key Technologies
 
 - **GitOps**: ArgoCD
-- **Kubernetes**: UpCloud Managed Kubernetes
+- **Kubernetes**: UpCloud Managed Kubernetes (multi-cluster: upc-dev, upc-prod)
 - **Ingress**: Traefik v2
 - **Certificates**: Cert-Manager + Let's Encrypt
 - **Policies**: Kyverno

docs/REFERENCE.md

@@ -21,7 +21,7 @@
 |-----------|-------|
 | **Provider** | UpCloud Managed Kubernetes |
 | **Environment** | Production (internal use) |
-| **Cluster Count** | Single cluster |
+| **Cluster Count** | Multi-cluster (upc-dev, upc-prod) |
 | **GitOps Tool** | ArgoCD |
 | **Ingress Controller** | Traefik v2 |
 | **Certificate Management** | Cert-Manager + Let's Encrypt |
@@ -71,7 +71,8 @@ Internet
 ```
 launchpad/
 ├── bootstrap.sh                   # Cluster initialization script
-├── _app-of-apps.yaml              # Root ArgoCD Application
+├── _app-of-apps-upc-dev.yaml     # Root ArgoCD Application (upc-dev)
+├── _app-of-apps-upc-prod.yaml    # Root ArgoCD Application (upc-prod)
 │
 ├── infra/                         # Infrastructure applications
 │   ├── cluster-resources-application.yaml
@@ -123,6 +124,7 @@ launchpad/
 │       ├── replicaset-cleaner.yaml
 │       ├── default-ns-blocker.yaml
 │       ├── secret-cloner.yaml
+│       ├── keycloak-client-cloner.yaml
 │       └── auth-sidecar-injector.yaml
 │
 ├── secrets/                       # Application secrets (sealed)
@@ -155,15 +157,15 @@ ArgoCd() {
   helm upgrade --install argocd argo-cd \
     --repo https://argoproj.github.io/argo-helm \
     --namespace argocd --create-namespace \
-    --values infra/values/argocd-values.yaml \
+    --values infra/values/base/argocd-values.yaml \
     --set notifications.context.clusterName="$CLUSTER_NAME" \
     --timeout 60s --atomic
 
-  kubectl apply -f _app-of-apps.yaml -n argocd
+  kubectl apply -f _app-of-apps-upc-dev.yaml -n argocd   # or _app-of-apps-upc-prod.yaml
 }
 ```
 
-**`_app-of-apps.yaml`**
+**`_app-of-apps-upc-dev.yaml`** / **`_app-of-apps-upc-prod.yaml`**
 ```yaml
 apiVersion: argoproj.io/v1alpha1
 kind: Application
@@ -614,7 +616,7 @@ retry:
 
 **Configuration**:
 ```yaml
-# infra/traefik-application.yaml
+# infra/base/traefik-application.yaml
 replicas: 2
 
 service:
@@ -789,7 +791,7 @@ persistence:
 
 **Configuration**:
 ```yaml
-# infra/gitea.yaml + infra/values/gitea-values.yaml
+# infra/base/gitea.yaml + infra/values/base/gitea-values.yaml
 ingress:
   host: git.forteapps.net
   tls: cert-manager (letsencrypt-prod)
@@ -832,7 +834,7 @@ postgresql:
 
 **Configuration**:
 ```yaml
-# infra/gitea-actions.yaml + infra/values/gitea-actions-values.yaml
+# infra/base/gitea-actions.yaml + infra/values/base/gitea-actions-values.yaml
 replicaCount: 3
 
 runner:
@@ -869,29 +871,44 @@ dind:
 - Gitea admin panel (`/admin/runners`) — runners show as Online
 - Create test workflow in `.gitea/workflows/test.yml` — job executes
 
-### Keycloak Secret Syncer
+### Keycloak Client Registrar
 
-**Type**: ArgoCD PostSync Job (deployed via Keycloak Helm chart `extraDeploy`)
+**Type**: CronJob (deployed via Keycloak Helm chart `extraDeploy`)
 **Namespace**: `keycloak`
+**Schedule**: `*/2 * * * *` (every 2 minutes)
 
-**Purpose**: Automatically extracts Keycloak-generated client secrets and syncs them into Kubernetes Secrets in target namespaces. Eliminates the need to manually manage OIDC client secrets.
+**Purpose**: Handles two responsibilities:
+1. **Legacy sync** — extracts secrets from Keycloak clients with `k8s.secret.sync: "true"` attribute (same as former PostSync syncer)
+2. **Self-service registration** — processes config Secrets (cloned by Kyverno) to register new OIDC clients and sync their credentials
 
 **How It Works**:
-1. Runs as an ArgoCD PostSync hook after Keycloak resources are healthy
+
-2. Authenticates to Keycloak Admin API using admin credentials from `keycloak-credentials` secret
+*Legacy path (existing clients like Gitea):*
-3. Queries all clients in the `forte` realm
+1. Authenticates to Keycloak Admin API using admin credentials from `keycloak-credentials` secret
-4. Filters clients with `k8s.secret.sync: "true"` attribute
+2. Queries all clients in the `forte` realm
-5. For each matching client, retrieves the auto-generated secret via Keycloak Admin API
+3. Filters clients with `k8s.secret.sync: "true"` attribute
-6. Creates/updates a K8s Secret in the target namespace (from `k8s.secret.namespace` attribute)
+4. For each matching client, retrieves the auto-generated secret via Keycloak Admin API
-7. Always writes a central copy to the `secrets` namespace (for external deployment retrieval)
+5. Creates/updates a K8s Secret in the target namespace (from `k8s.secret.namespace` attribute)
+6. Always writes a central copy to the `secrets` namespace
+
+*Self-service path (new clients):*
+1. Lists Secrets in `keycloak` namespace with label `keycloak.forteapps.net/client-config=true`
+2. For each config Secret, parses `client.json` and computes a config hash
+3. Skips if hash matches annotation and credential Secret already exists
+4. Creates or updates the Keycloak client via Admin API
+5. Fetches the generated client secret
+6. Upserts credential Secret in target namespace + central `secrets` namespace
+7. Annotates config Secret with sync status, config hash, and timestamp
 
 **Resources**:
-- `ServiceAccount`: `keycloak-secret-syncer` (namespace: `keycloak`)
+- `ServiceAccount`: `keycloak-client-registrar` (namespace: `keycloak`)
-- `ClusterRole`: `keycloak-secret-syncer` (secrets: get/create/update/patch; namespaces: get/list)
+- `ClusterRole`: `keycloak-client-registrar` (secrets: get/list/create/update/patch; namespaces: get/list)
-- `ClusterRoleBinding`: `keycloak-secret-syncer`
+- `ClusterRoleBinding`: `keycloak-client-registrar`
-- `Job`: `keycloak-secret-syncer` (PostSync hook)
+- `CronJob`: `keycloak-client-registrar`
 
-**Client Attributes** (set in `forte-realm.json`):
+**Kyverno Policy**: `keycloak-client-config-cloner` — clones labeled Secrets from app namespaces to `keycloak` namespace (see [Kyverno Policies](#kyverno-policies))
+
+**Legacy Client Attributes** (set in `forte-realm.json`):
 
 | Attribute | Required | Default | Description |
 |-----------|----------|---------|-------------|
@@ -901,31 +918,68 @@ dind:
 | `k8s.secret.client-id-key` | No | `client-id` | Field name for client ID in the Secret |
 | `k8s.secret.client-secret-key` | No | `client-secret` | Field name for client secret in the Secret |
 
-**Created Secret Format** (key names configurable via attributes):
+**Self-Service Config Secret Schema**:
 ```yaml
 apiVersion: v1
 kind: Secret
 metadata:
-  name: <k8s.secret.name>
+  name: keycloak-client-<app>
-  namespace: <k8s.secret.namespace>
+  namespace: <app-namespace>
   labels:
-    app.kubernetes.io/managed-by: keycloak-secret-syncer
+    keycloak.forteapps.net/client-config: "true"
+stringData:
+  client.json: |
+    {
+      "clientId": "<app>",
+      "name": "<App Name>",
+      "redirectUris": ["https://<app>.forteapps.net/*"],
+      "webOrigins": ["https://<app>.forteapps.net"],
+      "defaultClientScopes": ["openid", "email", "profile"],
+      "protocolMappers": [],
+      "secret": {
+        "namespace": "<app-namespace>",
+        "name": "<app>-oidc-credentials",
+        "keys": { "clientId": "client-id", "clientSecret": "client-secret" }
+      }
+    }
+```
+
+**Created Credential Secret Format**:
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: <target-name>
+  namespace: <target-namespace>
+  labels:
+    app.kubernetes.io/managed-by: keycloak-client-registrar
 type: Opaque
 data:
   <client-id-key>: <base64-encoded client ID>
   <client-secret-key>: <base64-encoded client secret>
 ```
 
+**Config Secret Annotations** (set by registrar):
+
+| Annotation | Description |
+|-----------|-------------|
+| `keycloak.forteapps.net/config-hash` | SHA-256 hash of client.json for change detection |
+| `keycloak.forteapps.net/sync-status` | `synced` or `error` |
+| `keycloak.forteapps.net/last-sync` | ISO 8601 timestamp of last successful sync |
+
 **Verification**:
 ```bash
-# Check job status
+# Check CronJob status
-kubectl get jobs -n keycloak
+kubectl get cronjobs -n keycloak
 
-# View syncer logs
+# View latest registrar logs
-kubectl logs -n keycloak job/keycloak-secret-syncer
+kubectl logs -n keycloak job/$(kubectl get jobs -n keycloak --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}')
 
 # Verify created secret
 kubectl get secret <name> -n <namespace> -o yaml
+
+# Check config Secret annotations (self-service)
+kubectl get secret keycloak-client-<app> -n keycloak -o jsonpath='{.metadata.annotations}'
 ```
 
 **See**: [Developer Guide - Adding a New Keycloak Client](DEVELOPER-GUIDE.md#adding-a-new-keycloak-client)
@@ -941,7 +995,7 @@ kubectl get secret <name> -n <namespace> -o yaml
 
 **Configuration**:
 ```yaml
-# infra/renovate.yaml + infra/values/renovate-values.yaml
+# infra/base/renovate.yaml + infra/values/base/renovate-values.yaml
 cronjob:
   schedule: "@daily"
   concurrencyPolicy: Forbid
@@ -1020,6 +1074,19 @@ spec:
 
 **Label Requirement**: Secrets must have `allowedToBeCloned: "true"`
 
+### Keycloak Client Config Cloner
+
+**File**: `cluster-resources/policies/keycloak-client-cloner.yaml`
+
+**Purpose**: Clones Secrets labeled `keycloak.forteapps.net/client-config: "true"` from app namespaces to the `keycloak` namespace. This allows apps to declare their OIDC client configuration in their own namespace, which the [Keycloak Client Registrar](#keycloak-client-registrar) then processes.
+
+**Trigger**: Any Secret with label `keycloak.forteapps.net/client-config: "true"` created outside the `keycloak` namespace.
+
+**Behavior**:
+- Generates a copy of the Secret in the `keycloak` namespace with the same name
+- Adds source tracking annotations (`keycloak.forteapps.net/source-namespace`, `keycloak.forteapps.net/source-name`)
+- `synchronize: true` — changes to the source Secret are reflected in the clone
+
 ### Default Namespace Blocker
 
 **File**: `cluster-resources/policies/default-ns-blocker.yaml`

infra/base/enterprise-apps.yaml

@@ -18,7 +18,7 @@ spec:
   source:
     repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD
-    path: apps
+    path: apps/overlays/upc-dev
   destination:
     server: https://kubernetes.default.svc
     namespace: apps

infra/base/fluent-bit.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: fluent-bit
       valueFiles:
-      - $values/infra/values/fluent-bit-values.yaml
+      - $values/infra/values/base/fluent-bit-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/gitea-actions.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: gitea-actions
       valueFiles:
-      - $values/infra/values/gitea-actions-values.yaml
+      - $values/infra/values/base/gitea-actions-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/gitea.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: gitea
       valueFiles:
-      - $values/infra/values/gitea-values.yaml
+      - $values/infra/values/base/gitea-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/grafana.yaml

@@ -21,7 +21,8 @@ spec:
     helm:
       releaseName: grafana
       valueFiles:
-      - $values/infra/values/grafana-values.yaml
+      - $values/infra/values/base/grafana-values.yaml
+      - $values/infra/values/upc-dev/grafana-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/keycloak.yaml

@@ -21,7 +21,8 @@ spec:
     helm:
       releaseName: keycloak
       valueFiles:
-      - $values/infra/values/keycloak-values.yaml
+      - $values/infra/values/base/keycloak-values.yaml
+      - $values/infra/values/upc-dev/keycloak-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD
@@ -43,6 +44,6 @@ spec:
 
   ignoreDifferences:
   - group: batch
-    kind: Job
+    kind: CronJob
     jsonPointers:
-    - /spec/template/spec/containers/0/args
+    - /spec/jobTemplate/spec/template/spec/containers/0/args

infra/base/kustomization.yaml

@@ -0,0 +1,23 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- traefik-application.yaml
+- keycloak.yaml
+- grafana.yaml
+- cert-manager-application.yaml
+- kyverno.yaml
+- sealedsecrets.yaml
+- prometheus.yaml
+- loki.yaml
+- fluent-bit.yaml
+- trivy.yaml
+- enterprise-apps.yaml
+- cluster-resources-application.yaml
+- kyverno-policies.yaml
+- secrets.yaml
+- gitea.yaml
+- gitea-actions.yaml
+- renovate.yaml
+- tempo.yaml
+- grafana-dashboards.yaml
+- network-policies-application.yaml

infra/base/loki.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: loki
       valueFiles:
-      - $values/infra/values/loki-values.yaml
+      - $values/infra/values/base/loki-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/prometheus.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: prometheus
       valueFiles:
-      - $values/infra/values/prometheus-values.yaml
+      - $values/infra/values/base/prometheus-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/renovate.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: renovate
       valueFiles:
-      - $values/infra/values/renovate-values.yaml
+      - $values/infra/values/base/renovate-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/secrets.yaml

@@ -18,7 +18,7 @@ spec:
   project: default
   source:
     repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
-    path: secrets
+    path: secrets/upc-dev
   destination:
     server: https://kubernetes.default.svc
     namespace: secrets

infra/base/tempo.yaml

@@ -21,7 +21,7 @@ spec:
     helm:
       releaseName: tempo
       valueFiles:
-      - $values/infra/values/tempo-values.yaml
+      - $values/infra/values/base/tempo-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/traefik-application.yaml

@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: traefik-system
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: traefik
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+  labels:
+    app.kubernetes.io/name: traefik
+    app.kubernetes.io/part-of: platform
+    app.kubernetes.io/managed-by: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+
+  sources:
+  - repoURL: https://traefik.github.io/charts
+    chart: traefik
+    targetRevision: "28.0.0"
+    helm:
+      releaseName: traefik
+      valueFiles:
+      - $values/infra/values/base/traefik-values.yaml
+      - $values/infra/values/upc-dev/traefik-values.yaml
+
+  - repoURL: git@github.com:fortedigital/sturdy-adventure.git
+    targetRevision: HEAD
+    ref: values
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: traefik-system
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+
+    syncOptions:
+    - CreateNamespace=true
+    - Validate=true
+    - ServerSideApply=true

infra/overlays/upc-dev/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+
+# No patches needed — base already has "upc-dev" paths
+# upc-dev is the default/base cluster

infra/overlays/upc-prod/kustomization.yaml

@@ -0,0 +1,50 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+
+patches:
+# Traefik: swap upc-dev → upc-prod in valueFiles
+- target:
+    kind: Application
+    name: traefik
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/traefik-values.yaml
+
+# Keycloak: swap upc-dev → upc-prod
+- target:
+    kind: Application
+    name: keycloak
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/keycloak-values.yaml
+
+# Grafana: swap upc-dev → upc-prod
+- target:
+    kind: Application
+    name: grafana
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/grafana-values.yaml
+
+# Secrets: change path to upc-prod
+- target:
+    kind: Application
+    name: secrets
+  patch: |
+    - op: replace
+      path: /spec/source/path
+      value: secrets/upc-prod
+
+# Enterprise-apps: point to upc-prod overlay
+- target:
+    kind: Application
+    name: enterprise-apps
+  patch: |
+    - op: replace
+      path: /spec/source/path
+      value: apps/overlays/upc-prod

infra/traefik-application.yaml

@@ -1,159 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: traefik-system
-  annotations:
-    argocd.argoproj.io/sync-wave: "-1"
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: traefik
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
-  labels:
-    app.kubernetes.io/name: traefik
-    app.kubernetes.io/part-of: platform
-    app.kubernetes.io/managed-by: argocd
-  finalizers:
-  - resources-finalizer.argocd.argoproj.io
-spec:
-  project: default
-
-  source:
-    repoURL: https://traefik.github.io/charts
-    chart: traefik
-    targetRevision: "28.0.0"
-
-    helm:
-      values: |
-        metrics:
-          addInternals: true
-        tracing:
-          otlp:
-            enabled: true
-        logs:
-          general:
-            level: DEBUG
-          access:
-            format: json
-            enabled: true
-
-        additionalArguments:
-          - "--tracing.otlp.http.endpoint=http://tempo.monitoring.svc.cluster.local:4318/v1/traces"
-
-        providers:
-          kubernetesIngress:
-            publishedService: # Fixes ArgoCD health checks for LoadBalancer services
-              enabled: true
-        deployment:
-          replicas: 2
-
-        ingressRoute:
-          dashboard:
-            enabled: true
-            # Optional: specify entrypoint
-            entrypoint: traefik
-
-        api:
-          dashboard: true
-          debug: false
-
-        service:
-          type: LoadBalancer
-          annotations:
-            traefik.ingress.kubernetes.io/router.entrypoints: websecure
-            traefik.ingress.kubernetes.io/router.priority: "42"
-            traefik.ingress.kubernetes.io/router.tls: "true"
-            service.beta.kubernetes.io/upcloud-load-balancer-config: |
-              {
-                  "frontends": [
-                      {
-                          "name": "web",
-                          "mode": "tcp"
-                      },
-                      {
-                          "name": "websecure",
-                          "mode": "tcp"
-                      },
-                      {
-                          "name": "giteassh",
-                          "mode": "tcp"
-                      }
-                  ],
-                  "backends": [
-                      {
-                          "name": "web",
-                          "properties": {
-                              "outbound_proxy_protocol": "v2"
-                          }
-                      },
-                      {
-                          "name": "websecure",
-                          "properties": {
-                              "outbound_proxy_protocol": "v2"
-                          }
-                      },
-                      {
-                          "name": "giteassh"
-                      }
-                  ]
-              }
-
-        ingressClass:
-          enabled: true
-          isDefaultClass: true
-
-        # Configure entry points
-        ports:
-          metrics:
-            expose: 
-              default: true
-            observability:
-              accessLogs: true
-              metrics: true
-              tracing: true
-              traceVerbosity: detailed
-          web:
-            proxyProtocol:
-              trustedIPs: "172.16.1.0/24"
-            forwardedHeaders:
-              trustedIPs: "172.16.1.0/24"
-            http:
-              redirections:
-                entrypoint:
-                  to: websecure
-                  scheme: https
-
-          websecure:
-            proxyProtocol:
-              trustedIPs: "172.16.1.0/24"
-            forwardedHeaders:
-              trustedIPs: "172.16.1.0/24"
-            observability:
-              accessLogs: true
-              metrics: true
-              tracing: true
-
-          giteassh:
-            port: 2222
-            expose:
-              default: true
-            exposedPort: 2222
-            protocol: TCP
-
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: traefik-system
-
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-      allowEmpty: false
-
-    syncOptions:
-    - CreateNamespace=true
-    - Validate=true
-    - ServerSideApply=true

infra/values/base/argocd-values.yaml

@@ -1,5 +1,3 @@
-global:
-  domain: argocd.127.0.0.1.nip.io
 configs:
   secret:
     createSecret: true
@@ -22,10 +20,6 @@ notifications:
   secret:
     create: false
 
-  # Shared context variables available in all templates
-  context:
-    clusterName: "dev-fd-no-svg1"
-
   # Define notification templates
   templates:
     template.app-syncing: |

infra/values/base/dot-ai-stack-values.yaml

@@ -0,0 +1,11 @@
+dot-ai:
+  ingress:
+    enabled: true
+    className: traefik
+dot-ai-ui:
+  uiAuth:
+    secretRef:
+      name: dot-ai-secrets
+  ingress:
+    enabled: true
+    className: traefik

infra/values/base/grafana-values.yaml

@@ -1,7 +1,5 @@
 ingress:
   enabled: true
-  hosts:
-  - grafana.127.0.0.1.nip.io
 resources:
   requests:
     cpu: 50m

infra/values/base/keycloak-values.yaml

@@ -0,0 +1,456 @@
+# Bitnami Keycloak Helm Chart Values
+# Chart version: 25.2.0
+
+image:
+  repository: bitnamilegacy/keycloak
+
+production: true
+proxyHeaders: xforwarded
+
+auth:
+  adminUser: admin
+  existingSecret: keycloak-credentials
+  passwordSecretKey: admin-password
+
+ingress:
+  enabled: true
+  tls: true
+  ingressClassName: traefik
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+
+metrics:
+  enabled: true
+  prometheusRule:
+    namespace: monitoring
+    enabled: true
+
+resources:
+  requests:
+    cpu: 250m
+    memory: 512Mi
+  limits:
+    cpu: 500m
+    memory: 1Gi
+
+postgresql:
+  enabled: true
+  image:
+    repository: bitnamilegacy/postgresql
+  auth:
+    existingSecret: keycloak-credentials
+    secretKeys:
+      adminPasswordKey: postgres-password
+      userPasswordKey: password
+    username: bn_keycloak
+    database: bitnami_keycloak
+  primary:
+    persistence:
+      size: 8Gi
+
+keycloakConfigCli:
+  enabled: true
+  image:
+    repository: bitnamilegacy/keycloak-config-cli
+  configuration:
+    forte-realm.json: |
+      {
+        "realm": "forte",
+        "enabled": true,
+        "displayName": "Forte",
+        "sslRequired": "external",
+        "registrationAllowed": false,
+        "loginWithEmailAllowed": true,
+        "resetPasswordAllowed": true,
+        "rememberMe": true,
+        "clients": [
+          {
+            "clientId": "gitea",
+            "name": "Gitea",
+            "enabled": true,
+            "protocol": "openid-connect",
+            "clientAuthenticatorType": "client-secret",
+            "standardFlowEnabled": true,
+            "directAccessGrantsEnabled": false,
+            "publicClient": false,
+            "redirectUris": ["https://git.forteapps.net/*"],
+            "webOrigins": ["https://git.forteapps.net"],
+            "defaultClientScopes": ["openid", "email", "profile"],
+            "attributes": {
+              "k8s.secret.sync": "true",
+              "k8s.secret.namespace": "gitea",
+              "k8s.secret.name": "gitea-oidc-credentials",
+              "k8s.secret.client-id-key": "key",
+              "k8s.secret.client-secret-key": "secret"
+            },
+            "protocolMappers": [
+              {
+                "name": "email_verified",
+                "protocol": "openid-connect",
+                "protocolMapper": "oidc-hardcoded-claim-mapper",
+                "config": {
+                  "claim.name": "email_verified",
+                  "claim.value": "true",
+                  "jsonType.label": "boolean",
+                  "id.token.claim": "true",
+                  "access.token.claim": "true",
+                  "userinfo.token.claim": "true"
+                }
+              }
+            ]
+          }
+        ]
+      }
+
+extraDeploy:
+# -- ServiceAccount for the client registrar CronJob
+- apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    name: keycloak-client-registrar
+    namespace: keycloak
+
+# -- ClusterRole granting access to secrets and namespaces
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: keycloak-client-registrar
+  rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list"]
+
+# -- ClusterRoleBinding for the registrar ServiceAccount
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: keycloak-client-registrar
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: keycloak-client-registrar
+  subjects:
+  - kind: ServiceAccount
+    name: keycloak-client-registrar
+    namespace: keycloak
+
+# -- CronJob: registers Keycloak clients and syncs secrets
+- apiVersion: batch/v1
+  kind: CronJob
+  metadata:
+    name: keycloak-client-registrar
+    namespace: keycloak
+  spec:
+    schedule: "*/2 * * * *"
+    concurrencyPolicy: Forbid
+    successfulJobsHistoryLimit: 1
+    failedJobsHistoryLimit: 3
+    jobTemplate:
+      spec:
+        backoffLimit: 3
+        template:
+          spec:
+            serviceAccountName: keycloak-client-registrar
+            restartPolicy: Never
+            containers:
+            - name: registrar
+              image: alpine:3.20
+              command: ["/bin/sh", "-c"]
+              args:
+              - |
+                set -e
+                apk add --no-cache curl jq > /dev/null 2>&1
+
+                KEYCLOAK_URL="http://keycloak:80"
+                REALM="forte"
+                K8S_API="https://kubernetes.default.svc"
+                SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+                CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+                CENTRAL_NS="secrets"
+
+                # --- Authenticate to Keycloak Admin API ---
+                ADMIN_USER="admin"
+                ADMIN_PASS=$(cat /secrets/admin-password)
+
+                echo "Authenticating to Keycloak..."
+                TOKEN=$(curl -sf -X POST "${KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" \
+                  -d "client_id=admin-cli" \
+                  -d "username=${ADMIN_USER}" \
+                  -d "password=${ADMIN_PASS}" \
+                  -d "grant_type=password" | jq -r '.access_token')
+
+                if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
+                  echo "ERROR: Failed to authenticate to Keycloak"
+                  exit 1
+                fi
+
+                # --- Helper functions ---
+
+                # Upsert a K8s Secret: try PUT (update), fall back to POST (create)
+                upsert_secret() {
+                  local ns="$1" name="$2" manifest="$3"
+                  local code
+                  code=$(curl -sf -o /dev/null -w "%{http_code}" \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    -H "Content-Type: application/json" \
+                    -X PUT -d "$manifest" \
+                    "${K8S_API}/api/v1/namespaces/${ns}/secrets/${name}")
+                  if [ "$code" = "200" ]; then
+                    echo "  Updated secret '${ns}/${name}'"
+                  elif [ "$code" = "404" ]; then
+                    code=$(curl -sf -o /dev/null -w "%{http_code}" \
+                      --cacert "$CA_CERT" \
+                      -H "Authorization: Bearer ${SA_TOKEN}" \
+                      -H "Content-Type: application/json" \
+                      -X POST -d "$manifest" \
+                      "${K8S_API}/api/v1/namespaces/${ns}/secrets")
+                    if [ "$code" = "201" ]; then
+                      echo "  Created secret '${ns}/${name}'"
+                    else
+                      echo "  ERROR: Failed to create secret '${ns}/${name}' (HTTP ${code})"
+                      return 1
+                    fi
+                  else
+                    echo "  ERROR: Failed to update secret '${ns}/${name}' (HTTP ${code})"
+                    return 1
+                  fi
+                }
+
+                # Build a credential Secret JSON manifest
+                build_credential_secret() {
+                  local ns="$1" name="$2" id_key="$3" secret_key="$4" b64_id="$5" b64_secret="$6"
+                  cat <<MANIFEST
+                {
+                  "apiVersion": "v1",
+                  "kind": "Secret",
+                  "metadata": {
+                    "name": "${name}",
+                    "namespace": "${ns}",
+                    "labels": {
+                      "app.kubernetes.io/managed-by": "keycloak-client-registrar"
+                    }
+                  },
+                  "type": "Opaque",
+                  "data": {
+                    "${id_key}": "${b64_id}",
+                    "${secret_key}": "${b64_secret}"
+                  }
+                }
+                MANIFEST
+                }
+
+                # Sync credentials to target + central namespace
+                sync_credentials() {
+                  local client_id="$1" client_uuid="$2" target_ns="$3" target_name="$4" id_key="$5" secret_key="$6"
+
+                  # Get the client secret from Keycloak
+                  local secret_value
+                  secret_value=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                    "${KEYCLOAK_URL}/admin/realms/${REALM}/clients/${client_uuid}/client-secret" \
+                    | jq -r '.value')
+
+                  if [ -z "$secret_value" ] || [ "$secret_value" = "null" ]; then
+                    echo "  WARNING: No secret found for client '${client_id}', skipping"
+                    return 0
+                  fi
+
+                  local b64_id b64_secret
+                  b64_id=$(printf '%s' "$client_id" | base64 | tr -d '\n')
+                  b64_secret=$(printf '%s' "$secret_value" | base64 | tr -d '\n')
+
+                  # Write to target namespace (if it exists)
+                  local ns_status
+                  ns_status=$(curl -sf -o /dev/null -w "%{http_code}" \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    "${K8S_API}/api/v1/namespaces/${target_ns}")
+
+                  if [ "$ns_status" = "200" ]; then
+                    local manifest
+                    manifest=$(build_credential_secret "$target_ns" "$target_name" "$id_key" "$secret_key" "$b64_id" "$b64_secret")
+                    upsert_secret "$target_ns" "$target_name" "$manifest" || return 1
+                  else
+                    echo "  WARNING: Namespace '${target_ns}' does not exist, skipping target"
+                  fi
+
+                  # Always write a central copy to the secrets namespace
+                  local central_manifest
+                  central_manifest=$(build_credential_secret "$CENTRAL_NS" "$target_name" "$id_key" "$secret_key" "$b64_id" "$b64_secret")
+                  upsert_secret "$CENTRAL_NS" "$target_name" "$central_manifest" || return 1
+                }
+
+                # Annotate a K8s Secret with sync status
+                annotate_secret() {
+                  local ns="$1" name="$2" key="$3" value="$4"
+                  local patch
+                  patch=$(printf '{"metadata":{"annotations":{"%s":"%s"}}}' "$key" "$value")
+                  curl -sf -o /dev/null \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    -H "Content-Type: application/strategic-merge-patch+json" \
+                    -X PATCH -d "$patch" \
+                    "${K8S_API}/api/v1/namespaces/${ns}/secrets/${name}"
+                }
+
+                # =============================================
+                # LEGACY PATH — sync existing realm clients
+                # =============================================
+                echo "=== Legacy sync: clients with k8s.secret.sync=true ==="
+
+                CLIENTS=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                  "${KEYCLOAK_URL}/admin/realms/${REALM}/clients")
+
+                SYNC_CLIENTS=$(echo "$CLIENTS" | jq -c '[.[] | select(.attributes["k8s.secret.sync"] == "true")]')
+                COUNT=$(echo "$SYNC_CLIENTS" | jq 'length')
+                echo "Found ${COUNT} legacy client(s) with sync enabled"
+
+                echo "$SYNC_CLIENTS" | jq -c '.[]' | while read -r CLIENT; do
+                  CLIENT_ID=$(echo "$CLIENT" | jq -r '.clientId')
+                  CLIENT_UUID=$(echo "$CLIENT" | jq -r '.id')
+                  TARGET_NS=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.namespace"]')
+                  TARGET_NAME=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.name"]')
+                  ID_KEY=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.client-id-key"] // "client-id"')
+                  SECRET_KEY=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.client-secret-key"] // "client-secret"')
+
+                  echo "Processing legacy client '${CLIENT_ID}' -> '${TARGET_NS}/${TARGET_NAME}' (keys: ${ID_KEY}, ${SECRET_KEY})"
+                  sync_credentials "$CLIENT_ID" "$CLIENT_UUID" "$TARGET_NS" "$TARGET_NAME" "$ID_KEY" "$SECRET_KEY"
+                done
+
+                # =============================================
+                # NEW PATH — self-service config Secrets
+                # =============================================
+                echo ""
+                echo "=== Self-service: config Secrets with label keycloak.forteapps.net/client-config=true ==="
+
+                CONFIG_SECRETS=$(curl -sf \
+                  --cacert "$CA_CERT" \
+                  -H "Authorization: Bearer ${SA_TOKEN}" \
+                  "${K8S_API}/api/v1/namespaces/keycloak/secrets?labelSelector=keycloak.forteapps.net/client-config=true")
+
+                CONFIG_COUNT=$(echo "$CONFIG_SECRETS" | jq '.items | length')
+                echo "Found ${CONFIG_COUNT} config Secret(s) to process"
+
+                echo "$CONFIG_SECRETS" | jq -c '.items[]' | while read -r CONFIG_SECRET; do
+                  CONFIG_NAME=$(echo "$CONFIG_SECRET" | jq -r '.metadata.name')
+                  SOURCE_NS=$(echo "$CONFIG_SECRET" | jq -r '.metadata.annotations["keycloak.forteapps.net/source-namespace"] // .metadata.labels["keycloak.forteapps.net/source-namespace"] // "unknown"')
+
+                  # Decode client.json from the Secret data
+                  CLIENT_JSON_B64=$(echo "$CONFIG_SECRET" | jq -r '.data["client.json"] // empty')
+                  if [ -z "$CLIENT_JSON_B64" ]; then
+                    echo "WARNING: Config Secret '${CONFIG_NAME}' missing client.json field, skipping"
+                    continue
+                  fi
+                  CLIENT_JSON=$(printf '%s' "$CLIENT_JSON_B64" | base64 -d)
+
+                  CLIENT_ID=$(echo "$CLIENT_JSON" | jq -r '.clientId')
+                  echo "Processing self-service client '${CLIENT_ID}' from config '${CONFIG_NAME}'"
+
+                  # Compute config hash for change detection
+                  CONFIG_HASH=$(printf '%s' "$CLIENT_JSON" | sha256sum | cut -d' ' -f1)
+                  EXISTING_HASH=$(echo "$CONFIG_SECRET" | jq -r '.metadata.annotations["keycloak.forteapps.net/config-hash"] // ""')
+
+                  # Extract secret delivery config from client.json
+                  CRED_NS=$(echo "$CLIENT_JSON" | jq -r '.secret.namespace // "'"${SOURCE_NS}"'"')
+                  CRED_NAME=$(echo "$CLIENT_JSON" | jq -r '.secret.name // "'"${CLIENT_ID}"'-oidc-credentials"')
+                  CRED_ID_KEY=$(echo "$CLIENT_JSON" | jq -r '.secret.keys.clientId // "client-id"')
+                  CRED_SECRET_KEY=$(echo "$CLIENT_JSON" | jq -r '.secret.keys.clientSecret // "client-secret"')
+
+                  # Check if credential Secret already exists in target namespace
+                  CRED_EXISTS=$(curl -sf -o /dev/null -w "%{http_code}" \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    "${K8S_API}/api/v1/namespaces/${CRED_NS}/secrets/${CRED_NAME}")
+
+                  # Skip if hash matches and credential Secret exists
+                  if [ "$CONFIG_HASH" = "$EXISTING_HASH" ] && [ "$CRED_EXISTS" = "200" ]; then
+                    echo "  No changes detected, skipping"
+                    continue
+                  fi
+
+                  # Build Keycloak client representation (strip our secret delivery config)
+                  KC_CLIENT=$(echo "$CLIENT_JSON" | jq '{
+                    clientId: .clientId,
+                    name: .name,
+                    enabled: true,
+                    protocol: "openid-connect",
+                    clientAuthenticatorType: "client-secret",
+                    standardFlowEnabled: true,
+                    directAccessGrantsEnabled: false,
+                    publicClient: false,
+                    redirectUris: .redirectUris,
+                    webOrigins: .webOrigins,
+                    defaultClientScopes: .defaultClientScopes,
+                    protocolMappers: (.protocolMappers // [])
+                  }')
+
+                  # Check if client already exists
+                  EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                    "${KEYCLOAK_URL}/admin/realms/${REALM}/clients?clientId=${CLIENT_ID}" \
+                    | jq -r '.[0].id // empty')
+
+                  if [ -n "$EXISTING" ]; then
+                    echo "  Updating existing Keycloak client (uuid: ${EXISTING})"
+                    HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" \
+                      -H "Authorization: Bearer ${TOKEN}" \
+                      -H "Content-Type: application/json" \
+                      -X PUT -d "$KC_CLIENT" \
+                      "${KEYCLOAK_URL}/admin/realms/${REALM}/clients/${EXISTING}")
+                    if [ "$HTTP_CODE" != "204" ] && [ "$HTTP_CODE" != "200" ]; then
+                      echo "  ERROR: Failed to update client '${CLIENT_ID}' (HTTP ${HTTP_CODE})"
+                      annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/sync-status" "error"
+                      continue
+                    fi
+                    CLIENT_UUID="$EXISTING"
+                  else
+                    echo "  Creating new Keycloak client '${CLIENT_ID}'"
+                    HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" \
+                      -H "Authorization: Bearer ${TOKEN}" \
+                      -H "Content-Type: application/json" \
+                      -X POST -d "$KC_CLIENT" \
+                      "${KEYCLOAK_URL}/admin/realms/${REALM}/clients")
+                    if [ "$HTTP_CODE" != "201" ]; then
+                      echo "  ERROR: Failed to create client '${CLIENT_ID}' (HTTP ${HTTP_CODE})"
+                      annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/sync-status" "error"
+                      continue
+                    fi
+                    # Fetch the newly created client's UUID
+                    CLIENT_UUID=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                      "${KEYCLOAK_URL}/admin/realms/${REALM}/clients?clientId=${CLIENT_ID}" \
+                      | jq -r '.[0].id')
+                  fi
+
+                  # Sync credentials to target namespace
+                  sync_credentials "$CLIENT_ID" "$CLIENT_UUID" "$CRED_NS" "$CRED_NAME" "$CRED_ID_KEY" "$CRED_SECRET_KEY"
+
+                  # Annotate config Secret with hash and sync status
+                  annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/config-hash" "$CONFIG_HASH"
+                  annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/sync-status" "synced"
+                  TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+                  annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/last-sync" "$TIMESTAMP"
+                  echo "  Synced successfully"
+                done
+
+                echo ""
+                echo "Client registrar run complete"
+              volumeMounts:
+              - name: keycloak-credentials
+                mountPath: /secrets
+                readOnly: true
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 200m
+                  memory: 128Mi
+            volumes:
+            - name: keycloak-credentials
+              secret:
+                secretName: keycloak-credentials
+                items:
+                - key: admin-password
+                  path: admin-password

infra/values/base/traefik-values.yaml

@@ -0,0 +1,50 @@
+providers:
+  kubernetesIngress:
+    publishedService: # Fixes ArgoCD health checks for LoadBalancer services
+      enabled: true
+deployment:
+  replicas: 2
+
+ingressRoute:
+  dashboard:
+    enabled: true
+    # Optional: specify entrypoint
+    entrypoint: traefik
+
+api:
+  dashboard: true
+  debug: false
+
+service:
+  type: LoadBalancer
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.priority: "42"
+    traefik.ingress.kubernetes.io/router.tls: "true"
+
+ingressClass:
+  enabled: true
+  isDefaultClass: true
+
+# Configure entry points
+ports:
+  metrics:
+    expose:
+      default: true
+    observability:
+      accessLogs: true
+      metrics: true
+      tracing: true
+      traceVerbosity: detailed
+  web:
+    http:
+      redirections:
+        entrypoint:
+          to: websecure
+          scheme: https
+
+  websecure:
+    observability:
+      accessLogs: true
+      metrics: true
+      tracing: true

infra/values/keycloak-values.yaml

@@ -1,317 +0,0 @@
-# Bitnami Keycloak Helm Chart Values
-# Host: id.forteapps.net
-# Chart version: 25.2.0
-
-image:
-  repository: bitnamilegacy/keycloak
-
-production: true
-proxyHeaders: xforwarded
-
-auth:
-  adminUser: admin
-  existingSecret: keycloak-credentials
-  passwordSecretKey: admin-password
-
-ingress:
-  enabled: true
-  hostname: id.forteapps.net
-  tls: true
-  ingressClassName: traefik
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
-
-metrics:
-  enabled: true
-  prometheusRule:
-    namespace: monitoring
-    enabled: true
-
-resources:
-  requests:
-    cpu: 250m
-    memory: 512Mi
-  limits:
-    cpu: 500m
-    memory: 1Gi
-
-postgresql:
-  enabled: true
-  image:
-    repository: bitnamilegacy/postgresql
-  auth:
-    existingSecret: keycloak-credentials
-    secretKeys:
-      adminPasswordKey: postgres-password
-      userPasswordKey: password
-    username: bn_keycloak
-    database: bitnami_keycloak
-  primary:
-    persistence:
-      size: 8Gi
-
-keycloakConfigCli:
-  enabled: true
-  image:
-    repository: bitnamilegacy/keycloak-config-cli
-  configuration:
-    forte-realm.json: |
-      {
-        "realm": "forte",
-        "enabled": true,
-        "displayName": "Forte",
-        "sslRequired": "external",
-        "registrationAllowed": false,
-        "loginWithEmailAllowed": true,
-        "resetPasswordAllowed": true,
-        "rememberMe": true,
-        "clients": [
-          {
-            "clientId": "gitea",
-            "name": "Gitea",
-            "enabled": true,
-            "protocol": "openid-connect",
-            "clientAuthenticatorType": "client-secret",
-            "standardFlowEnabled": true,
-            "directAccessGrantsEnabled": false,
-            "publicClient": false,
-            "redirectUris": ["https://git.forteapps.net/*"],
-            "webOrigins": ["https://git.forteapps.net"],
-            "defaultClientScopes": ["openid", "email", "profile"],
-            "attributes": {
-              "k8s.secret.sync": "true",
-              "k8s.secret.namespace": "gitea",
-              "k8s.secret.name": "gitea-oidc-credentials",
-              "k8s.secret.client-id-key": "key",
-              "k8s.secret.client-secret-key": "secret"
-            },
-            "protocolMappers": [
-              {
-                "name": "email_verified",
-                "protocol": "openid-connect",
-                "protocolMapper": "oidc-hardcoded-claim-mapper",
-                "config": {
-                  "claim.name": "email_verified",
-                  "claim.value": "true",
-                  "jsonType.label": "boolean",
-                  "id.token.claim": "true",
-                  "access.token.claim": "true",
-                  "userinfo.token.claim": "true"
-                }
-              }
-            ]
-          }
-        ]
-      }
-
-extraDeploy:
-# -- ServiceAccount for the secret syncer Job
-- apiVersion: v1
-  kind: ServiceAccount
-  metadata:
-    name: keycloak-secret-syncer
-    namespace: keycloak
-
-# -- ClusterRole granting access to secrets and namespaces
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: ClusterRole
-  metadata:
-    name: keycloak-secret-syncer
-  rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "create", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["namespaces"]
-    verbs: ["get", "list"]
-
-# -- ClusterRoleBinding for the syncer ServiceAccount
-- apiVersion: rbac.authorization.k8s.io/v1
-  kind: ClusterRoleBinding
-  metadata:
-    name: keycloak-secret-syncer
-  roleRef:
-    apiGroup: rbac.authorization.k8s.io
-    kind: ClusterRole
-    name: keycloak-secret-syncer
-  subjects:
-  - kind: ServiceAccount
-    name: keycloak-secret-syncer
-    namespace: keycloak
-
-# -- PostSync Job: extracts Keycloak client secrets into K8s Secrets
-- apiVersion: batch/v1
-  kind: Job
-  metadata:
-    name: keycloak-secret-syncer
-    namespace: keycloak
-    annotations:
-      argocd.argoproj.io/hook: PostSync
-      argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-  spec:
-    backoffLimit: 3
-    template:
-      spec:
-        serviceAccountName: keycloak-secret-syncer
-        restartPolicy: Never
-        containers:
-        - name: syncer
-          image: alpine:3.20
-          command: ["/bin/sh", "-c"]
-          args:
-          - |
-            set -e
-            apk add --no-cache curl jq > /dev/null 2>&1
-
-            KEYCLOAK_URL="http://keycloak:80"
-            REALM="forte"
-
-            # Read admin credentials from the keycloak-credentials secret
-            ADMIN_USER="admin"
-            ADMIN_PASS=$(cat /secrets/admin-password)
-
-            # Authenticate to Keycloak Admin API
-            echo "Authenticating to Keycloak..."
-            TOKEN=$(curl -sf -X POST "${KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" \
-              -d "client_id=admin-cli" \
-              -d "username=${ADMIN_USER}" \
-              -d "password=${ADMIN_PASS}" \
-              -d "grant_type=password" | jq -r '.access_token')
-
-            if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
-              echo "ERROR: Failed to authenticate to Keycloak"
-              exit 1
-            fi
-
-            # Get all clients in the realm
-            echo "Fetching clients from realm '${REALM}'..."
-            CLIENTS=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
-              "${KEYCLOAK_URL}/admin/realms/${REALM}/clients")
-
-            # Filter clients with k8s.secret.sync=true
-            SYNC_CLIENTS=$(echo "$CLIENTS" | jq -c '[.[] | select(.attributes["k8s.secret.sync"] == "true")]')
-            COUNT=$(echo "$SYNC_CLIENTS" | jq 'length')
-            echo "Found ${COUNT} client(s) with sync enabled"
-
-            K8S_API="https://kubernetes.default.svc"
-            SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
-            CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-            CENTRAL_NS="secrets"
-
-            # Upsert a K8s Secret: try PUT (update), fall back to POST (create)
-            upsert_secret() {
-              local ns="$1" name="$2" manifest="$3"
-              local code
-              code=$(curl -sf -o /dev/null -w "%{http_code}" \
-                --cacert "$CA_CERT" \
-                -H "Authorization: Bearer ${SA_TOKEN}" \
-                -H "Content-Type: application/json" \
-                -X PUT -d "$manifest" \
-                "${K8S_API}/api/v1/namespaces/${ns}/secrets/${name}")
-              if [ "$code" = "200" ]; then
-                echo "  Updated secret '${ns}/${name}'"
-              elif [ "$code" = "404" ]; then
-                code=$(curl -sf -o /dev/null -w "%{http_code}" \
-                  --cacert "$CA_CERT" \
-                  -H "Authorization: Bearer ${SA_TOKEN}" \
-                  -H "Content-Type: application/json" \
-                  -X POST -d "$manifest" \
-                  "${K8S_API}/api/v1/namespaces/${ns}/secrets")
-                if [ "$code" = "201" ]; then
-                  echo "  Created secret '${ns}/${name}'"
-                else
-                  echo "  ERROR: Failed to create secret '${ns}/${name}' (HTTP ${code})"
-                  return 1
-                fi
-              else
-                echo "  ERROR: Failed to update secret '${ns}/${name}' (HTTP ${code})"
-                return 1
-              fi
-            }
-
-            # Build a Secret JSON manifest
-            # Args: namespace, name, id-key, secret-key, b64-id, b64-secret
-            build_manifest() {
-              local ns="$1" name="$2" id_key="$3" secret_key="$4" b64_id="$5" b64_secret="$6"
-              cat <<MANIFEST
-            {
-              "apiVersion": "v1",
-              "kind": "Secret",
-              "metadata": {
-                "name": "${name}",
-                "namespace": "${ns}",
-                "labels": {
-                  "app.kubernetes.io/managed-by": "keycloak-secret-syncer"
-                }
-              },
-              "type": "Opaque",
-              "data": {
-                "${id_key}": "${b64_id}",
-                "${secret_key}": "${b64_secret}"
-              }
-            }
-            MANIFEST
-            }
-
-            echo "$SYNC_CLIENTS" | jq -c '.[]' | while read -r CLIENT; do
-              CLIENT_ID=$(echo "$CLIENT" | jq -r '.clientId')
-              CLIENT_UUID=$(echo "$CLIENT" | jq -r '.id')
-              TARGET_NS=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.namespace"]')
-              TARGET_NAME=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.name"]')
-
-              # Configurable key names (defaults: client-id, client-secret)
-              ID_KEY=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.client-id-key"] // "client-id"')
-              SECRET_KEY=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.client-secret-key"] // "client-secret"')
-
-              echo "Processing client '${CLIENT_ID}' -> secret '${TARGET_NS}/${TARGET_NAME}' (keys: ${ID_KEY}, ${SECRET_KEY})"
-
-              # Get the client secret from Keycloak
-              SECRET_VALUE=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
-                "${KEYCLOAK_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/client-secret" \
-                | jq -r '.value')
-
-              if [ -z "$SECRET_VALUE" ] || [ "$SECRET_VALUE" = "null" ]; then
-                echo "  WARNING: No secret found for client '${CLIENT_ID}', skipping"
-                continue
-              fi
-
-              B64_CLIENT_ID=$(printf '%s' "$CLIENT_ID" | base64 | tr -d '\n')
-              B64_SECRET=$(printf '%s' "$SECRET_VALUE" | base64 | tr -d '\n')
-
-              # 1. Write to target namespace (if it exists)
-              NS_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" \
-                --cacert "$CA_CERT" \
-                -H "Authorization: Bearer ${SA_TOKEN}" \
-                "${K8S_API}/api/v1/namespaces/${TARGET_NS}")
-
-              if [ "$NS_STATUS" = "200" ]; then
-                MANIFEST=$(build_manifest "$TARGET_NS" "$TARGET_NAME" "$ID_KEY" "$SECRET_KEY" "$B64_CLIENT_ID" "$B64_SECRET")
-                upsert_secret "$TARGET_NS" "$TARGET_NAME" "$MANIFEST" || exit 1
-              else
-                echo "  WARNING: Namespace '${TARGET_NS}' does not exist, skipping target"
-              fi
-
-              # 2. Always write a central copy to the secrets namespace
-              CENTRAL_MANIFEST=$(build_manifest "$CENTRAL_NS" "$TARGET_NAME" "$ID_KEY" "$SECRET_KEY" "$B64_CLIENT_ID" "$B64_SECRET")
-              upsert_secret "$CENTRAL_NS" "$TARGET_NAME" "$CENTRAL_MANIFEST" || exit 1
-            done
-
-            echo "Secret sync complete"
-          volumeMounts:
-          - name: keycloak-credentials
-            mountPath: /secrets
-            readOnly: true
-          resources:
-            requests:
-              cpu: 50m
-              memory: 64Mi
-            limits:
-              cpu: 200m
-              memory: 128Mi
-        volumes:
-        - name: keycloak-credentials
-          secret:
-            secretName: keycloak-credentials
-            items:
-            - key: admin-password
-              path: admin-password

infra/values/upc-dev/argocd-values.yaml

@@ -0,0 +1,5 @@
+global:
+  domain: argocd.127.0.0.1.nip.io
+notifications:
+  context:
+    clusterName: "dev-fd-eu-no-svg1"

infra/values/upc-dev/dot-ai-stack-values.yaml

@@ -0,0 +1,8 @@
+dot-ai:
+  ingress:
+    host: kubemcp.forteapps.net
+  webUI:
+    baseUrl: http://kubemcpui.forteapps.net
+dot-ai-ui:
+  ingress:
+    host: kubemcpui.forteapps.net

infra/values/upc-dev/grafana-values.yaml

@@ -0,0 +1,3 @@
+ingress:
+  hosts:
+  - grafana.forteapps.net

infra/values/upc-dev/keycloak-values.yaml

@@ -0,0 +1,2 @@
+ingress:
+  hostname: id.forteapps.net

infra/values/upc-dev/traefik-values.yaml

@@ -0,0 +1,40 @@
+service:
+  annotations:
+    service.beta.kubernetes.io/upcloud-load-balancer-config: |
+      {
+          "frontends": [
+              {
+                  "name": "web",
+                  "mode": "tcp"
+              },
+              {
+                  "name": "websecure",
+                  "mode": "tcp"
+              }
+          ],
+          "backends": [
+              {
+                  "name": "web",
+                  "properties": {
+                      "outbound_proxy_protocol": "v2"
+                  }
+              },
+              {
+                  "name": "websecure",
+                  "properties": {
+                      "outbound_proxy_protocol": "v2"
+                  }
+              }
+          ]
+      }
+ports:
+  web:
+    proxyProtocol:
+      trustedIPs: "172.16.1.0/24"
+    forwardedHeaders:
+      trustedIPs: "172.16.1.0/24"
+  websecure:
+    proxyProtocol:
+      trustedIPs: "172.16.1.0/24"
+    forwardedHeaders:
+      trustedIPs: "172.16.1.0/24"

infra/values/upc-prod/argocd-values.yaml

@@ -0,0 +1,5 @@
+global:
+  domain: argocd.us.forteapps.net
+notifications:
+  context:
+    clusterName: "dev-fd-us-east1"

infra/values/upc-prod/dot-ai-stack-values.yaml

@@ -0,0 +1,8 @@
+dot-ai:
+  ingress:
+    host: kubemcp.us.forteapps.net
+  webUI:
+    baseUrl: http://kubemcpui.us.forteapps.net
+dot-ai-ui:
+  ingress:
+    host: kubemcpui.us.forteapps.net

infra/values/upc-prod/grafana-values.yaml

@@ -0,0 +1,3 @@
+ingress:
+  hosts:
+  - grafana.us.forteapps.net

infra/values/upc-prod/keycloak-values.yaml

@@ -0,0 +1,2 @@
+ingress:
+  hostname: id.us.forteapps.net

infra/values/upc-prod/traefik-values.yaml

@@ -0,0 +1,13 @@
+service:
+  annotations: {}
+ports:
+  web:
+    proxyProtocol:
+      trustedIPs: "10.0.0.0/16"
+    forwardedHeaders:
+      trustedIPs: "10.0.0.0/16"
+  websecure:
+    proxyProtocol:
+      trustedIPs: "10.0.0.0/16"
+    forwardedHeaders:
+      trustedIPs: "10.0.0.0/16"

secrets/upc-dev/dot-ai-secrets.yaml

@@ -0,0 +1,18 @@
+# SealedSecret created after namespace (sync-wave: 0)
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: dot-ai-secrets
+  namespace: dot-ai
+spec:
+  encryptedData:
+    anthropic-api-key: AgASIRVNs7kIvZ/XwFJ4j9TJ05TW4YlFNzi2lx+6URlvzjTkMignK+y4/HD3wTK37BkIonnOXf/DGJcMgqgxOrHBVu9tLkSamve5qgf+SOZ8jxaqpLX2e5hdmHrgMp7rVWKiOoxM+bvu8Gdve3eMwXX8Eazj7W7vi430LsvW4BmPrQp+A9SOP+hwMlV5r+P5pUogC3hddt5KoSwpgnpOb8fRGlLDpc548eDNBJrhsAZiiFqc8+CycHX6idxWzlWOTPR05LBs7YrTIP/LUXWdyq6UG8yB5D0c9JNndnGo6kSfJxdaxCNO7GhMAOmo4CitzYn/zhICwuHPLKwUYQBcEHVpFxgub0U9fY7s4i0SAH9aF1kk8yqHWaCicHzi6B3AxMZYr+knS5vVNB+uGZmSKgZ7QNE1bF6E4Gg5bXRwk7OVQhuJSBzxVNDoXoJ6stL8ejU+MP5FtI792FxZbNDImu3kqor/t9wB49ZeB9Ngks2yvtlSrQ3G2lxJaOgNBPmKYhAX95JFI3xUSHOMa18ftn+aXmEQTXOHtM/IT8A95gUvIKz8Hrt03CwgZ8E+3lcsjAVsfNjqn/erqV+xi93OYOL6o9ivmTIem3MzsqjKuQ57HKaghnd+Ygdt/WfotRBSgZtCweOkZBV0XKxI1RmCQXk6dce8x4T7FDEeaSQwkc4yZ77LfQJnyVgU1rds9Hqv0RbymHK8JCQEMMb6PcK+Ow068fdj4NP27HkuYMB5JdPp682KIRwGznN2d3+QPXRd/ZHshnixudY7CLirlZl9xD0HssfSBI+uCrFEAln8D93UYbn7trF5gkKWYQsQ07CVZ5Z1qXieqwJWjHRY+oY=
+    auth-token: AgB65rUY3yJuTTLbGyrCtJPZp8UyEOr+kznSMGvUaZ9PoHq+kIEhazRdVKHa/WGvMWuc4x+XYRuIGj+2KiooyilUPrLcE3bz9fUOAFbRw3+iUh3WAgwK+f3eBUG6/0OoFw+GJ583IRidDW1t2BchMxSM1m+3vmQoF24qj7k9j/lWPsnX2IX3DhM0SomD1xmG+LsQMo2e4vQXB0BxhVDIQ621JTrvUPYzYx0NlPqZO/MtR1JWYS7WBTvegvgeBKLxfy9KLnqsuzu7rc2t9BYt7TRqvBg/prrKPdSV6Ei4GOZq+AcG15iOKXhsj8SxejPpM0QDemFTRQkdfz+k8ms4/SM0eylr5fEaa99TMTvjYGCfjJJyRcVm5Ef/XdmXiM3OI1u+9QNPiqXh1zmtp0yQMZ7nRE70kKQ2MHVhEmSUkBjovybIzLk4OL1v0FGDDqa1BN9KmNEBlo8H7DqXzfamVoNPyuuO625B3HgSeR3Udq8hngy9W/wiolmMNSc9C6vBA0HzE7Mkq79fHh/ruTi5zOJLfyEP7KQQqlNKfEtDFQfabaV9ERthQjuUs8ZIrdfCTOyQkrmmGY/sH5yodLRSYSEpHFrhPepwHS7lGzbVucW3Fn34D4OxJXyXQQObZKybLV7g6Oglf2Pdn11CEu/4+19RlykuOlm1dlj852a5NWWmmX319laLGlEd1BpG4cZyc9UZud68dGumwzrUupUMTLlJx8bR5rLgqaMpQMluwvq8MQbBXm4ySOcQ2jQhjw==
+    openai-api-key: AgBc4cSrd0riSxKPrp7AqrNYMDAZJTe3aCjTOwEb5pS/KARkgoEGz+0ZhLXa2bcCsPlQzqRzZ1c1cUCnBACkp1xoMfhSOoIJUWJQHRgZx5kEDhNIE/M8PQe8es7jYsW7/ui6iK8uj3Ljk21r8l/ctnP/KiKyML4553el28Ya7XsPQynRvpVexFC4BoJw50nkauLnvl3in0cKmEVAkMWorUP3Etapaj4DWkCQQ22RPN0xGNo9yiIUEAVE+uTRFNKGEHMIvJzgB291FJQtG/WoN/Sy/DeoZkUYndA7CbfFgiz9sq3GqJwyqTM+Ikzw6VCQ3TVWYO/6+yaEzT1NZ/rw00YhDyFoH+yGTkX5lrSUo8lGf3T9OODTdSS1203xtj4dhGbY70sPGMxDfucnO6piVcuRfh9bWg1VX+MTjqpBQE1J0DKJanhoh6lKbPOJhRbi0TIoCz1a3btbbKLDq2YJl7FXA3QNdBsR4qVJ3xmgWlH/eTUskCE2YxDzHmkxgZN2mL22SfeUJYtDRswRG0UGc6pvg2xrR0iEDB0UbH4FQK46fWv8aiOejteLlAAyA9HNA8yi73rTEjq55wpO19/MYj+6oUGEHpFaJje77INTHAKpdbfkp8iKotNXFYLM2hsjyau+x/AFjg87kEdIUVVHHLVMDhOq6NO+foM0nFOIv4lr2Yjl15+ImMSTEfcBr+nWaomnb9G8i5ptqz9bMHbxAcHpzWUBH9SZhzrCWZNDzOm2X2K6mpXhg4WX7VJMoIJk/f+bxTTKnWBawdnpCdbdG5GYQh7usqjPuELFYRTsx+6Tqoddp4KIEyMMxO81XObiN5vEBg74ygunxrjKNg5FoOkUA79YvcGBVVU1FNRl3d8IslXFqhEOT35nxGBB0T7ZORii34tZ2E551NkIo1WbCwilD3Dlgw==
+    ui-auth-token: AgDStP4jHhMehx/SAv5x2aQrJwChnWq8WhV/LVTSuuCSSy9kFQGa3Izyl21sMhLKUgrlS030GH50lbku+IY90S+MSBfaCq0Xb8qwohfH+qJulMRgljoU8r58/3ZsaBzbGJjiNMgBCwGlvuK85t3R3662ftwikahWqLfwAahi12L0llUDNyG9UB0Os3oR2CVAt83+YB07YspCWRzwuOz9D1SgOL/g/JF2hoL43pDD65BqYKdEuLFInJZx1Ul68V/FPmJK1gmJb/Kv8pgtk0sGTzxbbTdIQ1Ugf6+8//CWEq6GsMeEZG9VwExL/KYqobnbqd02FRgSSvWybhWLjiALLQvAFtce6FR6bur0rsariyogGjYCuXYdRnhf7QnB6NopwplhFP37Qf6E4q2pJTJ6Fzv0xq+XlfzNkn9Jvebrdkk/5CThW+wPAfx8r1VIySqBsYnPO/ZQDkfQ5ocX6fmzZ1iCg+0NI24k2YwCeMZtEAGpKEtejS0BLvqrilUG6Oz9sHCGHro4Bv5B/jzUjGbye0DSDj1f6c2RcpqMiUxfPvydJIcJGhrTx8sZS3qhEWME7kwjJCZpHEzfdv0weSJbgVGBSh9e1wjZxxeJGXuZKdFzdhpNEWP4uScGW0UnRDwxZzHMsLjS/W30mQmwmgJVTKdPl6VQrvdq7m4AC6+/d7iXlHazieC1KpBme4hWzO+/h7qRw1P5Va/ZWZtnGs8476J8hIojRtOJ/CdWwVLUa0gHo4CYnempIMwCIS3GtA==
+  template:
+    metadata:
+      creationTimestamp: null
+      name: dot-ai-secrets
+      namespace: dot-ai