feature/multicluster

Co-authored-by: Danijel Simeunovic <danijel.simeunovic@trumf.no>
Reviewed-on: #4
Reviewed-by: gitea_admin <admin@forteapps.net>

infra/values/base/argocd-values.yaml

@@ -0,0 +1,79 @@
+configs:
+  secret:
+    createSecret: true
+    argocdServerAdminPassword: "$2b$12$Tmb1jH7ADvwWoUoNPXXsfOf6JqEluqhq8mL06a8DGT2AP1GzbNsCm"
+  cm:
+    application.resourceTrackingMethod: annotation
+    timeout.reconciliation: 60s
+    admin.enabled: "true"
+  params:
+    "server.insecure": true
+server:
+  ingress:
+    enabled: false
+    ingressClassName: nginx
+  extraArgs:
+  - --insecure
+
+notifications:
+  # Don't create secret via Helm - using SealedSecret instead
+  secret:
+    create: false
+
+  # Define notification templates
+  templates:
+    template.app-syncing: |
+      webhook:
+        slack:
+          method: POST
+          body: |
+            {
+              "payload": "🖥️ {{ .context.clusterName }}: 🔄 *{{ .app.metadata.name }}* is syncing...\n📦 Revision: {{ .app.status.sync.revision | default `n/a` | substr 0 7 }}"
+            }
+    template.app-sync-succeeded: |
+      webhook:
+        slack:
+          method: POST
+          body: |
+            {
+              "payload": "🖥️ {{ .context.clusterName }}: ✅ *{{ .app.metadata.name }}* sync succeeded\n📦 Revision: {{ .app.status.sync.revision | default `n/a` | substr 0 7 }}{{ range .app.status.summary.images }}\n🏷️  Image: {{ . }}{{ end }}"
+            }
+    template.app-sync-failed: |
+      webhook:
+        slack:
+          method: POST
+          body: |
+            {
+              "payload": "🖥️ {{ .context.clusterName }}: ❌ *{{ .app.metadata.name }}* sync failed\n📦 Revision: {{ .app.status.sync.revision | default `n/a` | substr 0 7 }}\n⚠️  Message: {{ .app.status.operationState.message }}"
+            }
+    template.app-degraded: |
+      webhook:
+        slack:
+          method: POST
+          body: |
+            {
+              "payload": "🖥️ {{ .context.clusterName }}: ⚠️ *{{ .app.metadata.name }}* is degraded\n🏥 Health: {{ .app.status.health.status }}\n📦 Revision: {{ .app.status.sync.revision | default `n/a` | substr 0 7 }}{{ range .app.status.summary.images }}\n🏷️  Image: {{ . }}{{ end }}"
+            }
+
+  # Define notification triggers
+  triggers:
+    trigger.on-sync-running: |
+      - when: app.status.operationState.phase in ['Running']
+        send: [app-syncing]
+    trigger.on-sync-succeeded: |
+      - when: app.status.operationState.phase in ['Succeeded'] and app.status.health.status == 'Healthy'
+        send: [app-sync-succeeded]
+    trigger.on-sync-failed: |
+      - when: app.status.operationState.phase in ['Failed']
+        send: [app-sync-failed]
+    trigger.on-degraded: |
+      - when: app.status.health.status == 'Degraded'
+        send: [app-degraded]
+
+  # Define notification services (webhook for Slack)
+  notifiers:
+    service.webhook.slack: |
+      url: $slack-webhook-url
+      headers:
+      - name: Content-Type
+        value: application/json

infra/values/base/dot-ai-stack-values.yaml

@@ -0,0 +1,11 @@
+dot-ai:
+  ingress:
+    enabled: true
+    className: traefik
+dot-ai-ui:
+  uiAuth:
+    secretRef:
+      name: dot-ai-secrets
+  ingress:
+    enabled: true
+    className: traefik

infra/values/base/fluent-bit-values.yaml

@@ -0,0 +1,78 @@
+# Fluent Bit Helm Chart Values
+# Static configuration for Loki output
+
+config:
+  service: |
+    [SERVICE]
+        Daemon Off
+        Flush 1
+        Log_Level info
+        Parsers_File parsers.conf
+        Parsers_File custom_parsers.conf
+        HTTP_Server On
+        HTTP_Listen 0.0.0.0
+        HTTP_Port 2020
+        Health_Check On
+
+  inputs: |
+    [INPUT]
+        Name tail
+        Path /var/log/containers/*.log
+        multiline.parser docker, cri
+        Tag kube.*
+        Mem_Buf_Limit 5MB
+        Skip_Long_Lines On
+
+    [INPUT]
+        Name systemd
+        Tag host.*
+        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+        Read_From_Tail On
+
+  filters: |
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Kube_URL https://kubernetes.default.svc:443
+        Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
+        Kube_Tag_Prefix kube.var.log.containers.
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+
+  outputs: |
+    [OUTPUT]
+        Name loki
+        Match kube.*
+        Host loki-gateway.monitoring.svc.cluster.local
+        Port 80
+        Labels job=fluent-bit, namespace=$kubernetes['namespace_name'], pod=$kubernetes['pod_name'], container=$kubernetes['container_name'], stream=$stream
+        Auto_Kubernetes_Labels Off
+        Line_Format json
+
+    [OUTPUT]
+        Name loki
+        Match host.*
+        Host loki-gateway.monitoring.svc.cluster.local
+        Port 80
+        Labels job=fluent-bit-systemd
+        Line_Format json
+
+
+tolerations:
+- key: node-role.kubernetes.io/control-plane
+  operator: Exists
+  effect: NoSchedule
+- key: node-role.kubernetes.io/master
+  operator: Exists
+  effect: NoSchedule
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    cpu: 100m
+    memory: 256Mi

infra/values/base/gitea-actions-values.yaml

@@ -0,0 +1,36 @@
+## Gitea Act Runner - Helm values
+## Chart: actions v0.0.5 (https://dl.gitea.com/charts)
+
+enabled: true
+
+giteaRootURL: https://git.forteapps.net
+
+existingSecret: gitea-runner-token
+existingSecretKey: token
+
+statefulset:
+  replicas: 3
+
+  resources:
+    requests:
+      cpu: 250m
+      memory: 256Mi
+    limits:
+      cpu: "1"
+      memory: 1Gi
+
+  actRunner:
+    config: |
+      log:
+        level: info
+      cache:
+        enabled: false
+      container:
+        require_docker: true
+        docker_timeout: 300s
+      runner:
+        labels:
+          - "ubuntu-latest:docker://catthehacker/ubuntu:act-22.04"
+          - "ubuntu-22.04:docker://catthehacker/ubuntu:act-22.04"
+  dind:
+    rootless: false

infra/values/base/gitea-values.yaml

@@ -0,0 +1,181 @@
+# Gitea Helm Chart Values
+# Host: git.forteapps.net
+# Chart: gitea v12.5.0 (app v1.25.4)
+# Repo: https://dl.gitea.com/charts
+
+# -- Admin account (password from sealed secret)
+gitea:
+  admin:
+    existingSecret: gitea-credentials
+    email: admin@forteapps.net
+
+  # -- Gitea app.ini configuration
+  config:
+    APP_NAME: "Forte Git"
+
+    server:
+      DOMAIN: git.forteapps.net
+      ROOT_URL: https://git.forteapps.net
+      SSH_DOMAIN: git.forteapps.net
+      SSH_PORT: 2222
+      LFS_START_SERVER: true
+      ENABLE_GITEA_PAGES: true
+      ENABLE_BASIC_AUTH_CHALLENGE: true
+
+    service:
+      DISABLE_REGISTRATION: false
+      DEFAULT_ALLOW_CREATE_ORGANIZATION: false
+      REQUIRE_SIGNIN_VIEW: false
+      ALLOW_ONLY_EXTERNAL_REGISTRATION: true
+      ENABLE_BASIC_AUTHENTICATION: true
+      ENABLE_PASSWORD_SIGNIN_FORM: false
+      ENABLE_NOTIFY_MAIL: true
+
+    openid:
+      ENABLE_OPENID_SIGNIN: false
+      ENABLE_OPENID_SIGNUP: false
+
+    oauth2:
+      ENABLED: true
+      ENABLE_AUTO_REGISTRATION: true
+      USERNAME: email
+
+    session:
+      PROVIDER: db
+
+    cache:
+      ADAPTER: memory
+
+    database:
+      DB_TYPE: postgres
+
+    metrics:
+      ENABLED: true
+
+    repository:
+      DEFAULT_BRANCH: main
+      DEFAULT_PRIVATE: last
+
+    actions:
+      ENABLED: true
+
+    packages:
+      ENABLED: true
+
+    indexer:
+      ISSUE_INDEXER_TYPE: bleve
+      REPO_INDEXER_ENABLED: true
+
+    mailer:
+      ENABLED: true
+      PROTOCOL: smtp+starttls
+      SMTP_ADDR: smtp.office365.com
+      SMTP_PORT: 587
+      FROM: "noreply@fortedigital.com"
+
+    admin:
+      DEFAULT_EMAIL_NOTIFICATIONS: enabled
+
+  # -- SMTP credentials injected from secret (USER and PASSWD)
+  additionalConfigFromEnvs:
+  - name: GITEA__mailer__USER
+    valueFrom:
+      secretKeyRef:
+        name: gitea-smtp-secret
+        key: username
+  - name: GITEA__mailer__PASSWD
+    valueFrom:
+      secretKeyRef:
+        name: gitea-smtp-secret
+        key: password
+  # -- OIDC authentication via Forte
+  oauth:
+  - name: "Forte"
+    provider: "openidConnect"
+    existingSecret: gitea-oidc-credentials
+    key: gitea
+    autoDiscoverUrl: "https://id.forteapps.net/realms/forte/.well-known/openid-configuration"
+    scopes: "openid email profile organization"
+    groupClaimName: "groups"
+    adminGroup: ""
+    restrictedGroup: ""
+  # -- Prometheus metrics (scraped via annotations, no ServiceMonitor CRD needed)
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+
+# -- Ingress via Traefik with Let's Encrypt TLS
+ingress:
+  enabled: true
+  className: traefik
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+  hosts:
+  - host: git.forteapps.net
+    paths:
+    - path: /
+      pathType: Prefix
+  tls:
+  - secretName: gitea-tls
+    hosts:
+    - git.forteapps.net
+
+# -- Git repository storage
+persistence:
+  enabled: true
+  size: 10Gi
+  accessModes:
+  - ReadWriteOnce
+  storageClass: upcloud-block-storage-maxiops
+
+# -- Recreate strategy to avoid Multi-Attach errors with RWO volumes
+strategy:
+  type: Recreate
+
+# -- Pod resources
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+# -- Embedded PostgreSQL (Bitnami subchart)
+# Password auto-generated by the subchart; Gitea chart auto-wires the connection.
+postgresql:
+  enabled: true
+  auth:
+    username: gitea
+    database: gitea
+  primary:
+    persistence:
+      enabled: true
+      size: 8Gi
+      storageClass: upcloud-block-storage-maxiops
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 500m
+        memory: 512Mi
+
+# -- Disable PostgreSQL HA (using single-instance postgresql above)
+postgresql-ha:
+  enabled: false
+
+# -- Disable Redis cluster (use in-memory cache instead)
+redis-cluster:
+  enabled: false
+
+# -- Disable test pod
+test:
+  enabled: false
+
+# -- SSH service (ClusterIP, exposed externally via Traefik TCP IngressRoute on port 2222)
+service:
+  ssh:
+    type: ClusterIP
+    port: 22

infra/values/base/grafana-values.yaml

@@ -0,0 +1,76 @@
+ingress:
+  enabled: true
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    cpu: 100m
+    memory: 256Mi
+
+adminUser: admin
+adminPassword: "forte"
+
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+    - name: Prometheus
+      type: prometheus
+      url: http://prometheus-server.monitoring.svc.cluster.local
+      access: proxy
+      isDefault: true
+    - name: Loki
+      type: loki
+      uid: loki
+      url: http://loki-gateway.monitoring.svc.cluster.local
+      access: proxy
+    - name: Tempo
+      type: tempo
+      uid: tempo
+      url: http://tempo.monitoring.svc.cluster.local:3200
+      access: proxy
+      jsonData:
+        tracesToLogsV2:
+          datasourceUid: loki
+          tags:
+          - key: namespace
+          - key: pod
+          - key: container
+        tracesToMetrics:
+          datasourceUid: Prometheus
+          tags:
+          - key: service.name
+            value: service
+        nodeGraph:
+          enabled: true
+        serviceMap:
+          datasourceUid: Prometheus
+
+sidecar:
+  dashboards:
+    enabled: true
+    label: grafana_dashboard
+    labelValue: "1"
+    folder: /tmp/dashboards
+    provider:
+      foldersFromFilesStructure: false
+
+dashboardProviders:
+  dashboardproviders.yaml:
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      editable: true
+      options:
+        path: /var/lib/grafana/dashboards/default
+dashboards:
+  default:
+    kubernetes:
+      gnetId: 15758
+      revision: 1
+      datasource: Prometheus

infra/values/base/keycloak-values.yaml

@@ -0,0 +1,456 @@
+# Bitnami Keycloak Helm Chart Values
+# Chart version: 25.2.0
+
+image:
+  repository: bitnamilegacy/keycloak
+
+production: true
+proxyHeaders: xforwarded
+
+auth:
+  adminUser: admin
+  existingSecret: keycloak-credentials
+  passwordSecretKey: admin-password
+
+ingress:
+  enabled: true
+  tls: true
+  ingressClassName: traefik
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+
+metrics:
+  enabled: true
+  prometheusRule:
+    namespace: monitoring
+    enabled: true
+
+resources:
+  requests:
+    cpu: 250m
+    memory: 512Mi
+  limits:
+    cpu: 500m
+    memory: 1Gi
+
+postgresql:
+  enabled: true
+  image:
+    repository: bitnamilegacy/postgresql
+  auth:
+    existingSecret: keycloak-credentials
+    secretKeys:
+      adminPasswordKey: postgres-password
+      userPasswordKey: password
+    username: bn_keycloak
+    database: bitnami_keycloak
+  primary:
+    persistence:
+      size: 8Gi
+
+keycloakConfigCli:
+  enabled: true
+  image:
+    repository: bitnamilegacy/keycloak-config-cli
+  configuration:
+    forte-realm.json: |
+      {
+        "realm": "forte",
+        "enabled": true,
+        "displayName": "Forte",
+        "sslRequired": "external",
+        "registrationAllowed": false,
+        "loginWithEmailAllowed": true,
+        "resetPasswordAllowed": true,
+        "rememberMe": true,
+        "clients": [
+          {
+            "clientId": "gitea",
+            "name": "Gitea",
+            "enabled": true,
+            "protocol": "openid-connect",
+            "clientAuthenticatorType": "client-secret",
+            "standardFlowEnabled": true,
+            "directAccessGrantsEnabled": false,
+            "publicClient": false,
+            "redirectUris": ["https://git.forteapps.net/*"],
+            "webOrigins": ["https://git.forteapps.net"],
+            "defaultClientScopes": ["openid", "email", "profile"],
+            "attributes": {
+              "k8s.secret.sync": "true",
+              "k8s.secret.namespace": "gitea",
+              "k8s.secret.name": "gitea-oidc-credentials",
+              "k8s.secret.client-id-key": "key",
+              "k8s.secret.client-secret-key": "secret"
+            },
+            "protocolMappers": [
+              {
+                "name": "email_verified",
+                "protocol": "openid-connect",
+                "protocolMapper": "oidc-hardcoded-claim-mapper",
+                "config": {
+                  "claim.name": "email_verified",
+                  "claim.value": "true",
+                  "jsonType.label": "boolean",
+                  "id.token.claim": "true",
+                  "access.token.claim": "true",
+                  "userinfo.token.claim": "true"
+                }
+              }
+            ]
+          }
+        ]
+      }
+
+extraDeploy:
+# -- ServiceAccount for the client registrar CronJob
+- apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    name: keycloak-client-registrar
+    namespace: keycloak
+
+# -- ClusterRole granting access to secrets and namespaces
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: keycloak-client-registrar
+  rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list"]
+
+# -- ClusterRoleBinding for the registrar ServiceAccount
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: keycloak-client-registrar
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: keycloak-client-registrar
+  subjects:
+  - kind: ServiceAccount
+    name: keycloak-client-registrar
+    namespace: keycloak
+
+# -- CronJob: registers Keycloak clients and syncs secrets
+- apiVersion: batch/v1
+  kind: CronJob
+  metadata:
+    name: keycloak-client-registrar
+    namespace: keycloak
+  spec:
+    schedule: "*/2 * * * *"
+    concurrencyPolicy: Forbid
+    successfulJobsHistoryLimit: 1
+    failedJobsHistoryLimit: 3
+    jobTemplate:
+      spec:
+        backoffLimit: 3
+        template:
+          spec:
+            serviceAccountName: keycloak-client-registrar
+            restartPolicy: Never
+            containers:
+            - name: registrar
+              image: alpine:3.20
+              command: ["/bin/sh", "-c"]
+              args:
+              - |
+                set -e
+                apk add --no-cache curl jq > /dev/null 2>&1
+
+                KEYCLOAK_URL="http://keycloak:80"
+                REALM="forte"
+                K8S_API="https://kubernetes.default.svc"
+                SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+                CA_CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+                CENTRAL_NS="secrets"
+
+                # --- Authenticate to Keycloak Admin API ---
+                ADMIN_USER="admin"
+                ADMIN_PASS=$(cat /secrets/admin-password)
+
+                echo "Authenticating to Keycloak..."
+                TOKEN=$(curl -sf -X POST "${KEYCLOAK_URL}/realms/master/protocol/openid-connect/token" \
+                  -d "client_id=admin-cli" \
+                  -d "username=${ADMIN_USER}" \
+                  -d "password=${ADMIN_PASS}" \
+                  -d "grant_type=password" | jq -r '.access_token')
+
+                if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
+                  echo "ERROR: Failed to authenticate to Keycloak"
+                  exit 1
+                fi
+
+                # --- Helper functions ---
+
+                # Upsert a K8s Secret: try PUT (update), fall back to POST (create)
+                upsert_secret() {
+                  local ns="$1" name="$2" manifest="$3"
+                  local code
+                  code=$(curl -sf -o /dev/null -w "%{http_code}" \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    -H "Content-Type: application/json" \
+                    -X PUT -d "$manifest" \
+                    "${K8S_API}/api/v1/namespaces/${ns}/secrets/${name}")
+                  if [ "$code" = "200" ]; then
+                    echo "  Updated secret '${ns}/${name}'"
+                  elif [ "$code" = "404" ]; then
+                    code=$(curl -sf -o /dev/null -w "%{http_code}" \
+                      --cacert "$CA_CERT" \
+                      -H "Authorization: Bearer ${SA_TOKEN}" \
+                      -H "Content-Type: application/json" \
+                      -X POST -d "$manifest" \
+                      "${K8S_API}/api/v1/namespaces/${ns}/secrets")
+                    if [ "$code" = "201" ]; then
+                      echo "  Created secret '${ns}/${name}'"
+                    else
+                      echo "  ERROR: Failed to create secret '${ns}/${name}' (HTTP ${code})"
+                      return 1
+                    fi
+                  else
+                    echo "  ERROR: Failed to update secret '${ns}/${name}' (HTTP ${code})"
+                    return 1
+                  fi
+                }
+
+                # Build a credential Secret JSON manifest
+                build_credential_secret() {
+                  local ns="$1" name="$2" id_key="$3" secret_key="$4" b64_id="$5" b64_secret="$6"
+                  cat <<MANIFEST
+                {
+                  "apiVersion": "v1",
+                  "kind": "Secret",
+                  "metadata": {
+                    "name": "${name}",
+                    "namespace": "${ns}",
+                    "labels": {
+                      "app.kubernetes.io/managed-by": "keycloak-client-registrar"
+                    }
+                  },
+                  "type": "Opaque",
+                  "data": {
+                    "${id_key}": "${b64_id}",
+                    "${secret_key}": "${b64_secret}"
+                  }
+                }
+                MANIFEST
+                }
+
+                # Sync credentials to target + central namespace
+                sync_credentials() {
+                  local client_id="$1" client_uuid="$2" target_ns="$3" target_name="$4" id_key="$5" secret_key="$6"
+
+                  # Get the client secret from Keycloak
+                  local secret_value
+                  secret_value=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                    "${KEYCLOAK_URL}/admin/realms/${REALM}/clients/${client_uuid}/client-secret" \
+                    | jq -r '.value')
+
+                  if [ -z "$secret_value" ] || [ "$secret_value" = "null" ]; then
+                    echo "  WARNING: No secret found for client '${client_id}', skipping"
+                    return 0
+                  fi
+
+                  local b64_id b64_secret
+                  b64_id=$(printf '%s' "$client_id" | base64 | tr -d '\n')
+                  b64_secret=$(printf '%s' "$secret_value" | base64 | tr -d '\n')
+
+                  # Write to target namespace (if it exists)
+                  local ns_status
+                  ns_status=$(curl -sf -o /dev/null -w "%{http_code}" \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    "${K8S_API}/api/v1/namespaces/${target_ns}")
+
+                  if [ "$ns_status" = "200" ]; then
+                    local manifest
+                    manifest=$(build_credential_secret "$target_ns" "$target_name" "$id_key" "$secret_key" "$b64_id" "$b64_secret")
+                    upsert_secret "$target_ns" "$target_name" "$manifest" || return 1
+                  else
+                    echo "  WARNING: Namespace '${target_ns}' does not exist, skipping target"
+                  fi
+
+                  # Always write a central copy to the secrets namespace
+                  local central_manifest
+                  central_manifest=$(build_credential_secret "$CENTRAL_NS" "$target_name" "$id_key" "$secret_key" "$b64_id" "$b64_secret")
+                  upsert_secret "$CENTRAL_NS" "$target_name" "$central_manifest" || return 1
+                }
+
+                # Annotate a K8s Secret with sync status
+                annotate_secret() {
+                  local ns="$1" name="$2" key="$3" value="$4"
+                  local patch
+                  patch=$(printf '{"metadata":{"annotations":{"%s":"%s"}}}' "$key" "$value")
+                  curl -sf -o /dev/null \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    -H "Content-Type: application/strategic-merge-patch+json" \
+                    -X PATCH -d "$patch" \
+                    "${K8S_API}/api/v1/namespaces/${ns}/secrets/${name}"
+                }
+
+                # =============================================
+                # LEGACY PATH — sync existing realm clients
+                # =============================================
+                echo "=== Legacy sync: clients with k8s.secret.sync=true ==="
+
+                CLIENTS=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                  "${KEYCLOAK_URL}/admin/realms/${REALM}/clients")
+
+                SYNC_CLIENTS=$(echo "$CLIENTS" | jq -c '[.[] | select(.attributes["k8s.secret.sync"] == "true")]')
+                COUNT=$(echo "$SYNC_CLIENTS" | jq 'length')
+                echo "Found ${COUNT} legacy client(s) with sync enabled"
+
+                echo "$SYNC_CLIENTS" | jq -c '.[]' | while read -r CLIENT; do
+                  CLIENT_ID=$(echo "$CLIENT" | jq -r '.clientId')
+                  CLIENT_UUID=$(echo "$CLIENT" | jq -r '.id')
+                  TARGET_NS=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.namespace"]')
+                  TARGET_NAME=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.name"]')
+                  ID_KEY=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.client-id-key"] // "client-id"')
+                  SECRET_KEY=$(echo "$CLIENT" | jq -r '.attributes["k8s.secret.client-secret-key"] // "client-secret"')
+
+                  echo "Processing legacy client '${CLIENT_ID}' -> '${TARGET_NS}/${TARGET_NAME}' (keys: ${ID_KEY}, ${SECRET_KEY})"
+                  sync_credentials "$CLIENT_ID" "$CLIENT_UUID" "$TARGET_NS" "$TARGET_NAME" "$ID_KEY" "$SECRET_KEY"
+                done
+
+                # =============================================
+                # NEW PATH — self-service config Secrets
+                # =============================================
+                echo ""
+                echo "=== Self-service: config Secrets with label keycloak.forteapps.net/client-config=true ==="
+
+                CONFIG_SECRETS=$(curl -sf \
+                  --cacert "$CA_CERT" \
+                  -H "Authorization: Bearer ${SA_TOKEN}" \
+                  "${K8S_API}/api/v1/namespaces/keycloak/secrets?labelSelector=keycloak.forteapps.net/client-config=true")
+
+                CONFIG_COUNT=$(echo "$CONFIG_SECRETS" | jq '.items | length')
+                echo "Found ${CONFIG_COUNT} config Secret(s) to process"
+
+                echo "$CONFIG_SECRETS" | jq -c '.items[]' | while read -r CONFIG_SECRET; do
+                  CONFIG_NAME=$(echo "$CONFIG_SECRET" | jq -r '.metadata.name')
+                  SOURCE_NS=$(echo "$CONFIG_SECRET" | jq -r '.metadata.annotations["keycloak.forteapps.net/source-namespace"] // .metadata.labels["keycloak.forteapps.net/source-namespace"] // "unknown"')
+
+                  # Decode client.json from the Secret data
+                  CLIENT_JSON_B64=$(echo "$CONFIG_SECRET" | jq -r '.data["client.json"] // empty')
+                  if [ -z "$CLIENT_JSON_B64" ]; then
+                    echo "WARNING: Config Secret '${CONFIG_NAME}' missing client.json field, skipping"
+                    continue
+                  fi
+                  CLIENT_JSON=$(printf '%s' "$CLIENT_JSON_B64" | base64 -d)
+
+                  CLIENT_ID=$(echo "$CLIENT_JSON" | jq -r '.clientId')
+                  echo "Processing self-service client '${CLIENT_ID}' from config '${CONFIG_NAME}'"
+
+                  # Compute config hash for change detection
+                  CONFIG_HASH=$(printf '%s' "$CLIENT_JSON" | sha256sum | cut -d' ' -f1)
+                  EXISTING_HASH=$(echo "$CONFIG_SECRET" | jq -r '.metadata.annotations["keycloak.forteapps.net/config-hash"] // ""')
+
+                  # Extract secret delivery config from client.json
+                  CRED_NS=$(echo "$CLIENT_JSON" | jq -r '.secret.namespace // "'"${SOURCE_NS}"'"')
+                  CRED_NAME=$(echo "$CLIENT_JSON" | jq -r '.secret.name // "'"${CLIENT_ID}"'-oidc-credentials"')
+                  CRED_ID_KEY=$(echo "$CLIENT_JSON" | jq -r '.secret.keys.clientId // "client-id"')
+                  CRED_SECRET_KEY=$(echo "$CLIENT_JSON" | jq -r '.secret.keys.clientSecret // "client-secret"')
+
+                  # Check if credential Secret already exists in target namespace
+                  CRED_EXISTS=$(curl -sf -o /dev/null -w "%{http_code}" \
+                    --cacert "$CA_CERT" \
+                    -H "Authorization: Bearer ${SA_TOKEN}" \
+                    "${K8S_API}/api/v1/namespaces/${CRED_NS}/secrets/${CRED_NAME}")
+
+                  # Skip if hash matches and credential Secret exists
+                  if [ "$CONFIG_HASH" = "$EXISTING_HASH" ] && [ "$CRED_EXISTS" = "200" ]; then
+                    echo "  No changes detected, skipping"
+                    continue
+                  fi
+
+                  # Build Keycloak client representation (strip our secret delivery config)
+                  KC_CLIENT=$(echo "$CLIENT_JSON" | jq '{
+                    clientId: .clientId,
+                    name: .name,
+                    enabled: true,
+                    protocol: "openid-connect",
+                    clientAuthenticatorType: "client-secret",
+                    standardFlowEnabled: true,
+                    directAccessGrantsEnabled: false,
+                    publicClient: false,
+                    redirectUris: .redirectUris,
+                    webOrigins: .webOrigins,
+                    defaultClientScopes: .defaultClientScopes,
+                    protocolMappers: (.protocolMappers // [])
+                  }')
+
+                  # Check if client already exists
+                  EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                    "${KEYCLOAK_URL}/admin/realms/${REALM}/clients?clientId=${CLIENT_ID}" \
+                    | jq -r '.[0].id // empty')
+
+                  if [ -n "$EXISTING" ]; then
+                    echo "  Updating existing Keycloak client (uuid: ${EXISTING})"
+                    HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" \
+                      -H "Authorization: Bearer ${TOKEN}" \
+                      -H "Content-Type: application/json" \
+                      -X PUT -d "$KC_CLIENT" \
+                      "${KEYCLOAK_URL}/admin/realms/${REALM}/clients/${EXISTING}")
+                    if [ "$HTTP_CODE" != "204" ] && [ "$HTTP_CODE" != "200" ]; then
+                      echo "  ERROR: Failed to update client '${CLIENT_ID}' (HTTP ${HTTP_CODE})"
+                      annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/sync-status" "error"
+                      continue
+                    fi
+                    CLIENT_UUID="$EXISTING"
+                  else
+                    echo "  Creating new Keycloak client '${CLIENT_ID}'"
+                    HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" \
+                      -H "Authorization: Bearer ${TOKEN}" \
+                      -H "Content-Type: application/json" \
+                      -X POST -d "$KC_CLIENT" \
+                      "${KEYCLOAK_URL}/admin/realms/${REALM}/clients")
+                    if [ "$HTTP_CODE" != "201" ]; then
+                      echo "  ERROR: Failed to create client '${CLIENT_ID}' (HTTP ${HTTP_CODE})"
+                      annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/sync-status" "error"
+                      continue
+                    fi
+                    # Fetch the newly created client's UUID
+                    CLIENT_UUID=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+                      "${KEYCLOAK_URL}/admin/realms/${REALM}/clients?clientId=${CLIENT_ID}" \
+                      | jq -r '.[0].id')
+                  fi
+
+                  # Sync credentials to target namespace
+                  sync_credentials "$CLIENT_ID" "$CLIENT_UUID" "$CRED_NS" "$CRED_NAME" "$CRED_ID_KEY" "$CRED_SECRET_KEY"
+
+                  # Annotate config Secret with hash and sync status
+                  annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/config-hash" "$CONFIG_HASH"
+                  annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/sync-status" "synced"
+                  TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+                  annotate_secret "keycloak" "$CONFIG_NAME" "keycloak.forteapps.net/last-sync" "$TIMESTAMP"
+                  echo "  Synced successfully"
+                done
+
+                echo ""
+                echo "Client registrar run complete"
+              volumeMounts:
+              - name: keycloak-credentials
+                mountPath: /secrets
+                readOnly: true
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 200m
+                  memory: 128Mi
+            volumes:
+            - name: keycloak-credentials
+              secret:
+                secretName: keycloak-credentials
+                items:
+                - key: admin-password
+                  path: admin-password

infra/values/base/loki-values.yaml

@@ -0,0 +1,44 @@
+global:
+  dnsService: coredns
+
+deploymentMode: SingleBinary
+loki:
+  auth_enabled: false
+  commonConfig:
+    replication_factor: 1
+  storage:
+    type: 'filesystem'
+  schemaConfig:
+    configs:
+    - from: "2024-01-01"
+      store: tsdb
+      index:
+        prefix: loki_index_
+        period: 24h
+      object_store: filesystem # we're storing on filesystem so there's no real persistence here.
+      schema: v13
+  limits_config:
+    reject_old_samples: true
+    reject_old_samples_max_age: 168h
+    ingestion_rate_mb: 15
+    ingestion_burst_size_mb: 30
+    max_line_size: 512KB
+chunksCache:
+  enabled: false
+resultsCache:
+  enabled: false
+singleBinary:
+  replicas: 1
+  resources:
+    requests:
+      cpu: 50m
+      memory: 256Mi
+    limits:
+      cpu: 100m
+      memory: 1Gi
+read:
+  replicas: 0
+backend:
+  replicas: 0
+write:
+  replicas: 0

infra/values/base/opencost-values.yaml

@@ -0,0 +1,30 @@
+opencost:
+  exporter:
+    defaultClusterId: launchpad
+    extraEnv:
+      EMIT_KSM_V1_METRICS: "false"
+      EMIT_KSM_V1_METRICS_ONLY: "true"
+    prometheus:
+      internal:
+        enabled: true
+        serviceName: prometheus-server
+        namespaceName: monitoring
+        port: 80
+    customPricing:
+      enabled: true
+      provider: custom
+      costModel:
+        description: "UpCloud 4-node cluster pricing"
+        CPU: "5.86"
+        RAM: "1.46"
+        GPU: "0"
+        storage: "0.34"
+        zoneNetworkEgress: "0"
+        regionNetworkEgress: "0"
+        internetNetworkEgress: "0"
+  ui:
+    enabled: false
+service:
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9003"

infra/values/base/prometheus-values.yaml

@@ -0,0 +1,81 @@
+server:
+  ingress:
+    enabled: false
+  service:
+    servicePort: 80
+
+  resources:
+    requests:
+      cpu: 150m
+      memory: 512Mi
+    limits:
+      cpu: 300m
+      memory: 1Gi
+
+  enableLifecycle: true
+  extraFlags:
+  - web.enable-remote-write-receiver
+
+extraScrapeConfigs: |
+  - job_name: kyverno
+    scrape_interval: 15s
+    metrics_path: /metrics
+    kubernetes_sd_configs:
+      - role: endpoints
+        namespaces:
+          names:
+            - kyverno
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_endpoint_port_name]
+        regex: metrics-port
+        action: keep
+      - source_labels: [__meta_kubernetes_service_name]
+        target_label: service
+      - source_labels: [__meta_kubernetes_pod_name]
+        target_label: pod
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: namespace
+
+  - job_name: trivy-operator
+    scrape_interval: 30s
+    metrics_path: /metrics
+    kubernetes_sd_configs:
+      - role: pod
+        namespaces:
+          names:
+            - trivy-system
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        regex: trivy-operator
+        action: keep
+      - source_labels: [__meta_kubernetes_pod_container_port_number]
+        regex: "8080"
+        action: keep
+      - source_labels: [__meta_kubernetes_pod_name]
+        target_label: pod
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: namespace
+      - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance]
+        target_label: instance
+
+  - job_name: traefik
+    scrape_interval: 15s
+    metrics_path: /metrics
+    kubernetes_sd_configs:
+      - role: endpoints
+        namespaces:
+          names:
+            - traefik-system
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_endpoint_port_name]
+        regex: metrics
+        action: keep
+      - source_labels: [__meta_kubernetes_service_name]
+        target_label: service
+      - source_labels: [__meta_kubernetes_pod_name]
+        target_label: pod
+      - source_labels: [__meta_kubernetes_namespace]
+        target_label: namespace
+
+alertmanager:
+  enabled: false

infra/values/base/renovate-values.yaml

@@ -0,0 +1,45 @@
+cronjob:
+  schedule: "@daily"
+  concurrencyPolicy: Forbid
+
+renovate:
+  config: |
+    {
+      "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+      "platform": "gitea",
+      "endpoint": "https://git.forteapps.net",
+      "autodiscover": true,
+      "gitAuthor": "Renovate Bot <renovate@forteapps.net>",
+      "packageRules": [
+        {
+          "matchRepositories": ["**/10x"],
+          "assignees": ["edvard.unsvag"],
+          "reviewers": ["edvard.unsvag"]
+        },
+        {
+          "matchRepositories": ["**/auth-sidecar"],
+          "assignees": ["danijel.simeunovic"],
+          "reviewers": ["danijel.simeunovic"]
+        },
+        {
+          "matchRepositories": ["**/forte-helm"],
+          "assignees": ["danijel.simeunovic"],
+          "reviewers": ["danijel.simeunovic"]
+        }
+      ]
+    }
+
+envFrom:
+- secretRef:
+    name: renovate-env
+
+env:
+  LOG_LEVEL: info
+
+resources:
+  requests:
+    cpu: 500m
+    memory: 1Gi
+  limits:
+    cpu: "2"
+    memory: 4Gi

infra/values/base/tempo-values.yaml

@@ -0,0 +1,34 @@
+tempo:
+  metricsGenerator:
+    enabled: true
+    remoteWriteUrl: "http://prometheus-server.monitoring.svc.cluster.local/api/v1/write"
+  overrides:
+    defaults:
+      metrics_generator:
+        processors:
+        - service-graphs
+        - span-metrics
+  storage:
+    trace:
+      backend: local
+      local:
+        path: /var/tempo/traces
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: "0.0.0.0:4317"
+        http:
+          endpoint: "0.0.0.0:4318"
+
+persistence:
+  enabled: true
+  size: 10Gi
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 256Mi
+  limits:
+    cpu: 100m
+    memory: 512Mi

infra/values/base/traefik-values.yaml

@@ -0,0 +1,50 @@
+providers:
+  kubernetesIngress:
+    publishedService: # Fixes ArgoCD health checks for LoadBalancer services
+      enabled: true
+deployment:
+  replicas: 2
+
+ingressRoute:
+  dashboard:
+    enabled: true
+    # Optional: specify entrypoint
+    entrypoint: traefik
+
+api:
+  dashboard: true
+  debug: false
+
+service:
+  type: LoadBalancer
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.priority: "42"
+    traefik.ingress.kubernetes.io/router.tls: "true"
+
+ingressClass:
+  enabled: true
+  isDefaultClass: true
+
+# Configure entry points
+ports:
+  metrics:
+    expose:
+      default: true
+    observability:
+      accessLogs: true
+      metrics: true
+      tracing: true
+      traceVerbosity: detailed
+  web:
+    http:
+      redirections:
+        entrypoint:
+          to: websecure
+          scheme: https
+
+  websecure:
+    observability:
+      accessLogs: true
+      metrics: true
+      tracing: true