multi-cloud + mcp

cluster-resources/gitea-backup-cronjob.yaml

@@ -57,17 +57,17 @@ spec:
                 - sh
                 - -c
                 - |
-                  mc alias set upcloud "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
+                  mc alias set s3 "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
 
                   TIMESTAMP=$(date +%Y%m%d-%H%M%S)
                   KEY="gitea-dump-${TIMESTAMP}.zip"
                   echo "Uploading ${KEY}..."
-                  mc cp /backup/gitea-dump.zip "upcloud/${S3_BUCKET}/${KEY}" && \
+                  mc cp /backup/gitea-dump.zip "s3/${S3_BUCKET}/${KEY}" && \
                   echo "Upload complete."
 
                   # Prune backups older than 7 days
                   echo "Pruning backups older than 7 days..."
-                  mc rm --older-than 7d --force "upcloud/${S3_BUCKET}/" 2>&1 || true
+                  mc rm --older-than 7d --force "s3/${S3_BUCKET}/" 2>&1 || true
                   echo "Pruning complete."
               envFrom:
                 - secretRef:

docs/REFERENCE.md

@@ -9,6 +9,7 @@
 - [Kyverno Policies](#kyverno-policies)
 - [Configuration Reference](#configuration-reference)
 - [API Endpoints](#api-endpoints)
+- [Cloud Overlay Pattern](#cloud-overlay-pattern)
 - [Glossary](#glossary)
 
 ---
@@ -92,16 +93,34 @@ launchpad/
 │   ├── sealedsecrets.yaml
 │   ├── secrets.yaml
 │   ├── renovate.yaml
+│   ├── base/                      # ArgoCD Application manifests (Kustomize base)
+│   │   ├── gitea.yaml
+│   │   ├── opencost.yaml
+│   │   ├── traefik-application.yaml
+│   │   ├── keycloak.yaml
+│   │   ├── grafana.yaml
+│   │   └── ...
+│   ├── overlays/
+│   │   └── upc-prod/
+│   │       └── kustomization.yaml # Patches upc-dev → upc-prod valueFile paths
 │   └── values/
-│       ├── argocd-values.yaml
+│       ├── base/                   # Cloud-agnostic Helm values
-│       ├── prometheus-values.yaml
+│       │   ├── gitea-values.yaml
+│       │   ├── opencost-values.yaml
+│       │   ├── prometheus-values.yaml
+│       │   └── ...
+│       ├── upc-dev/               # UpCloud dev overlay values
+│       │   ├── traefik-values.yaml
+│       │   ├── keycloak-values.yaml
+│       │   ├── grafana-values.yaml
+│       │   ├── gitea-values.yaml
+│       │   └── opencost-values.yaml
+│       └── upc-prod/              # UpCloud prod overlay values
+│           ├── traefik-values.yaml
+│           ├── keycloak-values.yaml
 │           ├── grafana-values.yaml
-│       ├── loki-values.yaml
-│       ├── tempo-values.yaml
 │           ├── gitea-values.yaml
-│       ├── gitea-actions-values.yaml
+│           └── opencost-values.yaml
-│       ├── fluent-bit-values.yaml
-│       └── renovate-values.yaml
 │
 ├── apps/                          # Business applications
 │   ├── mcp10x.yaml
@@ -135,6 +154,15 @@ launchpad/
 │   ├── mcp10x-credentials-sealed.yaml
 │   └── musicman-credentials.yaml
 │
+├── scripts/                       # Operational helper scripts
+│   ├── gitea-backup.sh            # S3 backup helper (list/download)
+│   ├── gitea-restore.sh
+│   └── backup/                    # Per-cloud backup reference scripts
+│       ├── s3-minio.sh            # S3-compatible (UpCloud, MinIO, Wasabi)
+│       ├── aws-s3.sh              # Native AWS S3
+│       ├── azure-blob.sh          # Azure Blob Storage
+│       └── gcp-gcs.sh             # GCP Cloud Storage
+│
 ├── private/                       # Local-only (Git-ignored)
 │   ├── *.yaml
 │   └── *.sh
@@ -1621,6 +1649,79 @@ POST /loki/api/v1/push
 
 ---
 
+## Cloud Overlay Pattern
+
+### Overview
+
+Cloud-specific configuration (StorageClass, LoadBalancer annotations, pricing models, etc.) lives in per-cloud overlay value files, **not** in `base/`. This means adding a new cloud provider (AKS, EKS, GKE) only requires a new overlay directory — no base changes.
+
+### How It Works
+
+Each ArgoCD Application uses **multi-source Helm values** with two value files:
+
+```yaml
+# infra/base/gitea.yaml (example)
+helm:
+  valueFiles:
+  - $values/infra/values/base/gitea-values.yaml      # [0] cloud-agnostic
+  - $values/infra/values/upc-dev/gitea-values.yaml    # [1] cloud-specific (default: upc-dev)
+```
+
+The `upc-prod` Kustomize overlay patches index `[1]` to swap the cloud-specific file:
+
+```yaml
+# infra/overlays/upc-prod/kustomization.yaml
+- target:
+    kind: Application
+    name: gitea
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/gitea-values.yaml
+```
+
+### Components Using Cloud Overlays
+
+| Component | Cloud-specific config | Overlay value file |
+|-----------|----------------------|-------------------|
+| **Traefik** | LB annotations, proxy protocol IPs | `traefik-values.yaml` |
+| **Keycloak** | Hostname, TLS settings | `keycloak-values.yaml` |
+| **Grafana** | Hostname, datasource URLs | `grafana-values.yaml` |
+| **Gitea** | StorageClass (persistence + PostgreSQL) | `gitea-values.yaml` |
+| **OpenCost** | Custom pricing model (CPU/RAM/storage rates) | `opencost-values.yaml` |
+
+### Backup CronJob
+
+The `gitea-backup` CronJob uses a generic `s3` alias for `minio/mc`. The actual endpoint and credentials come from the `gitea-backup-s3` Sealed Secret, which is per-cloud. Reference scripts for different cloud providers are in `scripts/backup/`:
+
+| Script | Provider | Tool |
+|--------|----------|------|
+| `s3-minio.sh` | S3-compatible (UpCloud, MinIO, Wasabi) | `minio/mc` |
+| `aws-s3.sh` | AWS S3 | `aws` CLI |
+| `azure-blob.sh` | Azure Blob Storage | `az` CLI |
+| `gcp-gcs.sh` | GCP Cloud Storage | `gsutil` |
+
+### Adding a New Cloud Provider
+
+To add support for a new cloud (e.g., `aks-dev`):
+
+1. **Create overlay value directory**: `infra/values/aks-dev/`
+2. **Add cloud-specific value files** for each component that needs one:
+   - `traefik-values.yaml` — LB annotations, proxy protocol config
+   - `keycloak-values.yaml` — hostname/TLS if different
+   - `grafana-values.yaml` — hostname/datasources if different
+   - `gitea-values.yaml` — `storageClass` for persistence + PostgreSQL
+   - `opencost-values.yaml` — `customPricing` cost model for your cloud
+3. **Create a Kustomize overlay** (if needed): `infra/overlays/aks-prod/kustomization.yaml`
+   - Patch each Application's `valueFiles[1]` to point to `aks-prod/` files
+4. **Create a root Application**: `_app-of-apps-aks-dev.yaml` pointing to the overlay
+5. **Create Sealed Secrets** for the new cluster:
+   - `secrets/aks-dev/` — TLS certs, credentials, backup S3 config
+6. **Update `gitea-backup-s3` secret** with the new cloud's S3-compatible endpoint
+7. **Bootstrap**: `kubectl apply -f _app-of-apps-aks-dev.yaml -n argocd`
+
+---
+
 ## Glossary
 
 ### Terms

infra/base/gitea.yaml

@@ -22,6 +22,7 @@ spec:
       releaseName: gitea
       valueFiles:
       - $values/infra/values/base/gitea-values.yaml
+      - $values/infra/values/upc-dev/gitea-values.yaml
 
   - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
     targetRevision: HEAD

infra/base/opencost.yaml

@@ -22,6 +22,7 @@ spec:
       releaseName: opencost
       valueFiles:
       - $values/infra/values/base/opencost-values.yaml
+      - $values/infra/values/upc-dev/opencost-values.yaml
 
   - repoURL: git@github.com:fortedigital/sturdy-adventure.git
     targetRevision: HEAD

infra/overlays/upc-prod/kustomization.yaml

@@ -31,6 +31,24 @@ patches:
       path: /spec/sources/0/helm/valueFiles/1
       value: $values/infra/values/upc-prod/grafana-values.yaml
 
+# Gitea: swap upc-dev → upc-prod
+- target:
+    kind: Application
+    name: gitea
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/gitea-values.yaml
+
+# OpenCost: swap upc-dev → upc-prod
+- target:
+    kind: Application
+    name: opencost
+  patch: |
+    - op: replace
+      path: /spec/sources/0/helm/valueFiles/1
+      value: $values/infra/values/upc-prod/opencost-values.yaml
+
 # Secrets: change path to upc-prod
 - target:
     kind: Application

infra/values/base/gitea-values.yaml

@@ -130,7 +130,6 @@ persistence:
   size: 10Gi
   accessModes:
   - ReadWriteOnce
-  storageClass: upcloud-block-storage-maxiops
 
 # -- Recreate strategy to avoid Multi-Attach errors with RWO volumes
 strategy:
@@ -156,7 +155,6 @@ postgresql:
     persistence:
       enabled: true
       size: 8Gi
-      storageClass: upcloud-block-storage-maxiops
     resources:
       requests:
         cpu: 100m

infra/values/base/opencost-values.yaml

@@ -10,18 +10,6 @@ opencost:
         serviceName: prometheus-server
         namespaceName: monitoring
         port: 80
-    customPricing:
-      enabled: true
-      provider: custom
-      costModel:
-        description: "UpCloud 4-node cluster pricing"
-        CPU: "5.86"
-        RAM: "1.46"
-        GPU: "0"
-        storage: "0.34"
-        zoneNetworkEgress: "0"
-        regionNetworkEgress: "0"
-        internetNetworkEgress: "0"
   ui:
     enabled: false
 service:

infra/values/upc-dev/gitea-values.yaml

@@ -0,0 +1,7 @@
+# UpCloud-specific: block storage class for Gitea + PostgreSQL
+persistence:
+  storageClass: upcloud-block-storage-maxiops
+postgresql:
+  primary:
+    persistence:
+      storageClass: upcloud-block-storage-maxiops

infra/values/upc-dev/opencost-values.yaml

@@ -0,0 +1,15 @@
+# UpCloud-specific: custom pricing model
+opencost:
+  exporter:
+    customPricing:
+      enabled: true
+      provider: custom
+      costModel:
+        description: "UpCloud 4-node cluster pricing"
+        CPU: "5.86"
+        RAM: "1.46"
+        GPU: "0"
+        storage: "0.34"
+        zoneNetworkEgress: "0"
+        regionNetworkEgress: "0"
+        internetNetworkEgress: "0"

infra/values/upc-prod/gitea-values.yaml

@@ -0,0 +1,7 @@
+# UpCloud-specific: block storage class for Gitea + PostgreSQL
+persistence:
+  storageClass: upcloud-block-storage-maxiops
+postgresql:
+  primary:
+    persistence:
+      storageClass: upcloud-block-storage-maxiops

infra/values/upc-prod/opencost-values.yaml

@@ -0,0 +1,15 @@
+# UpCloud-specific: custom pricing model
+opencost:
+  exporter:
+    customPricing:
+      enabled: true
+      provider: custom
+      costModel:
+        description: "UpCloud 4-node cluster pricing"
+        CPU: "5.86"
+        RAM: "1.46"
+        GPU: "0"
+        storage: "0.34"
+        zoneNetworkEgress: "0"
+        regionNetworkEgress: "0"
+        internetNetworkEgress: "0"

scripts/backup/aws-s3.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# AWS S3 backup upload (native AWS CLI)
+# Uses: aws cli v2
+# Env: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION, S3_BUCKET
+
+BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+KEY="gitea-dump-${TIMESTAMP}.zip"
+
+echo "Uploading ${KEY}..."
+aws s3 cp "$BACKUP_FILE" "s3://${S3_BUCKET}/${KEY}"
+echo "Upload complete."
+
+# Prune backups older than 7 days
+echo "Pruning backups older than 7 days..."
+CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%S)
+aws s3api list-objects-v2 --bucket "${S3_BUCKET}" --query "Contents[?LastModified<'${CUTOFF}'].Key" --output text \
+  | tr '\t' '\n' \
+  | while read -r key; do
+      [ -n "$key" ] && aws s3 rm "s3://${S3_BUCKET}/${key}" && echo "Deleted: ${key}"
+    done
+echo "Pruning complete."

scripts/backup/azure-blob.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# Azure Blob Storage backup upload
+# Uses: az cli
+# Env: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, AZURE_CONTAINER
+
+BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+KEY="gitea-dump-${TIMESTAMP}.zip"
+
+echo "Uploading ${KEY}..."
+az storage blob upload \
+  --account-name "${AZURE_STORAGE_ACCOUNT}" \
+  --account-key "${AZURE_STORAGE_KEY}" \
+  --container-name "${AZURE_CONTAINER}" \
+  --name "${KEY}" \
+  --file "$BACKUP_FILE" \
+  --overwrite
+echo "Upload complete."
+
+# Prune backups older than 7 days
+echo "Pruning backups older than 7 days..."
+CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%SZ)
+az storage blob list \
+  --account-name "${AZURE_STORAGE_ACCOUNT}" \
+  --account-key "${AZURE_STORAGE_KEY}" \
+  --container-name "${AZURE_CONTAINER}" \
+  --query "[?properties.lastModified<'${CUTOFF}'].name" -o tsv \
+  | while read -r name; do
+      [ -n "$name" ] && az storage blob delete \
+        --account-name "${AZURE_STORAGE_ACCOUNT}" \
+        --account-key "${AZURE_STORAGE_KEY}" \
+        --container-name "${AZURE_CONTAINER}" \
+        --name "$name" && echo "Deleted: ${name}"
+    done
+echo "Pruning complete."

scripts/backup/gcp-gcs.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# GCP Cloud Storage backup upload
+# Uses: gsutil (gcloud SDK)
+# Env: GCS_BUCKET (e.g. gs://my-bucket)
+
+BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+KEY="gitea-dump-${TIMESTAMP}.zip"
+
+echo "Uploading ${KEY}..."
+gsutil cp "$BACKUP_FILE" "${GCS_BUCKET}/${KEY}"
+echo "Upload complete."
+
+# Prune backups older than 7 days — GCS lifecycle rules are preferred,
+# but this works as a manual fallback
+echo "Pruning backups older than 7 days..."
+CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%SZ)
+gsutil ls -l "${GCS_BUCKET}/" \
+  | grep 'gitea-dump-' \
+  | while read -r size date name; do
+      if [[ "$date" < "$CUTOFF" ]]; then
+        gsutil rm "$name" && echo "Deleted: ${name}"
+      fi
+    done
+echo "Pruning complete."

scripts/backup/s3-minio.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# S3-compatible backup upload (UpCloud Objects, MinIO, Wasabi, etc.)
+# Uses: minio/mc
+# Env: S3_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET
+
+BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+KEY="gitea-dump-${TIMESTAMP}.zip"
+
+mc alias set s3 "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
+
+echo "Uploading ${KEY}..."
+mc cp "$BACKUP_FILE" "s3/${S3_BUCKET}/${KEY}"
+echo "Upload complete."
+
+# Prune backups older than 7 days
+echo "Pruning backups older than 7 days..."
+mc rm --older-than 7d --force "s3/${S3_BUCKET}/" 2>&1 || true
+echo "Pruning complete."

scripts/gitea-backup.sh

@@ -13,7 +13,7 @@ NAMESPACE="gitea"
 SECRET="gitea-backup-s3"
 IMAGE="minio/mc:latest"
 POD_NAME="gitea-backup-helper"
-ALIAS_CMD='mc alias set upcloud ${S3_ENDPOINT} ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} > /dev/null'
+ALIAS_CMD='mc alias set s3 ${S3_ENDPOINT} ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} > /dev/null'
 
 cleanup() {
   kubectl -n "$NAMESPACE" delete pod "$POD_NAME" --ignore-not-found --grace-period=0 > /dev/null 2>&1 || true
@@ -41,7 +41,7 @@ mc_run() {
 case "${1:-help}" in
   list)
     echo "Listing backups..."
-    mc_run 'mc ls upcloud/${S3_BUCKET}/'
+    mc_run 'mc ls s3/${S3_BUCKET}/'
     ;;
 
   download)
@@ -49,7 +49,7 @@ case "${1:-help}" in
 
     if [ "$FILE" = "latest" ]; then
       echo "Finding latest backup..."
-      FILE=$(mc_run 'mc ls upcloud/${S3_BUCKET}/' | sort | tail -1 | awk '{print $NF}' | tr -d '[:space:]')
+      FILE=$(mc_run 'mc ls s3/${S3_BUCKET}/' | sort | tail -1 | awk '{print $NF}' | tr -d '[:space:]')
       if [ -z "$FILE" ]; then
         echo "No backups found."
         exit 1
@@ -74,7 +74,7 @@ case "${1:-help}" in
     kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$POD_NAME" --timeout=60s > /dev/null 2>&1
 
     echo "Saving to ./$FILE ..."
-    kubectl -n "$NAMESPACE" exec "$POD_NAME" -- sh -c "${ALIAS_CMD} && mc cat upcloud/\${S3_BUCKET}/$FILE" > "./$FILE"
+    kubectl -n "$NAMESPACE" exec "$POD_NAME" -- sh -c "${ALIAS_CMD} && mc cat s3/\${S3_BUCKET}/$FILE" > "./$FILE"
     cleanup
 
     echo "Downloaded: ./$FILE"