From e0bdaab4223209955dd07c385b3c1f421c36da93 Mon Sep 17 00:00:00 2001 From: Danijel Simeunovic Date: Wed, 22 Apr 2026 13:34:48 +0200 Subject: [PATCH] multi-cloud + mcp --- cluster-resources/gitea-backup-cronjob.yaml | 6 +- docs/REFERENCE.md | 119 ++++++++++++++++++-- infra/base/gitea.yaml | 1 + infra/base/opencost.yaml | 1 + infra/overlays/upc-prod/kustomization.yaml | 18 +++ infra/values/base/gitea-values.yaml | 2 - infra/values/base/opencost-values.yaml | 12 -- infra/values/upc-dev/gitea-values.yaml | 7 ++ infra/values/upc-dev/opencost-values.yaml | 15 +++ infra/values/upc-prod/gitea-values.yaml | 7 ++ infra/values/upc-prod/opencost-values.yaml | 15 +++ scripts/backup/aws-s3.sh | 23 ++++ scripts/backup/azure-blob.sh | 36 ++++++ scripts/backup/gcp-gcs.sh | 26 +++++ scripts/backup/s3-minio.sh | 20 ++++ scripts/gitea-backup.sh | 8 +- 16 files changed, 286 insertions(+), 30 deletions(-) create mode 100644 infra/values/upc-dev/gitea-values.yaml create mode 100644 infra/values/upc-dev/opencost-values.yaml create mode 100644 infra/values/upc-prod/gitea-values.yaml create mode 100644 infra/values/upc-prod/opencost-values.yaml create mode 100644 scripts/backup/aws-s3.sh create mode 100644 scripts/backup/azure-blob.sh create mode 100644 scripts/backup/gcp-gcs.sh create mode 100644 scripts/backup/s3-minio.sh diff --git a/cluster-resources/gitea-backup-cronjob.yaml b/cluster-resources/gitea-backup-cronjob.yaml index d05ec17..e8a6fa4 100644 --- a/cluster-resources/gitea-backup-cronjob.yaml +++ b/cluster-resources/gitea-backup-cronjob.yaml @@ -57,17 +57,17 @@ spec: - sh - -c - | - mc alias set upcloud "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" + mc alias set s3 "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" TIMESTAMP=$(date +%Y%m%d-%H%M%S) KEY="gitea-dump-${TIMESTAMP}.zip" echo "Uploading ${KEY}..." - mc cp /backup/gitea-dump.zip "upcloud/${S3_BUCKET}/${KEY}" && \ + mc cp /backup/gitea-dump.zip "s3/${S3_BUCKET}/${KEY}" && \ echo "Upload complete." # Prune backups older than 7 days echo "Pruning backups older than 7 days..." - mc rm --older-than 7d --force "upcloud/${S3_BUCKET}/" 2>&1 || true + mc rm --older-than 7d --force "s3/${S3_BUCKET}/" 2>&1 || true echo "Pruning complete." envFrom: - secretRef: diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 0ba2013..4bab669 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -9,6 +9,7 @@ - [Kyverno Policies](#kyverno-policies) - [Configuration Reference](#configuration-reference) - [API Endpoints](#api-endpoints) +- [Cloud Overlay Pattern](#cloud-overlay-pattern) - [Glossary](#glossary) --- @@ -92,16 +93,34 @@ launchpad/ │ ├── sealedsecrets.yaml │ ├── secrets.yaml │ ├── renovate.yaml +│ ├── base/ # ArgoCD Application manifests (Kustomize base) +│ │ ├── gitea.yaml +│ │ ├── opencost.yaml +│ │ ├── traefik-application.yaml +│ │ ├── keycloak.yaml +│ │ ├── grafana.yaml +│ │ └── ... +│ ├── overlays/ +│ │ └── upc-prod/ +│ │ └── kustomization.yaml # Patches upc-dev → upc-prod valueFile paths │ └── values/ -│ ├── argocd-values.yaml -│ ├── prometheus-values.yaml -│ ├── grafana-values.yaml -│ ├── loki-values.yaml -│ ├── tempo-values.yaml -│ ├── gitea-values.yaml -│ ├── gitea-actions-values.yaml -│ ├── fluent-bit-values.yaml -│ └── renovate-values.yaml +│ ├── base/ # Cloud-agnostic Helm values +│ │ ├── gitea-values.yaml +│ │ ├── opencost-values.yaml +│ │ ├── prometheus-values.yaml +│ │ └── ... +│ ├── upc-dev/ # UpCloud dev overlay values +│ │ ├── traefik-values.yaml +│ │ ├── keycloak-values.yaml +│ │ ├── grafana-values.yaml +│ │ ├── gitea-values.yaml +│ │ └── opencost-values.yaml +│ └── upc-prod/ # UpCloud prod overlay values +│ ├── traefik-values.yaml +│ ├── keycloak-values.yaml +│ ├── grafana-values.yaml +│ ├── gitea-values.yaml +│ └── opencost-values.yaml │ ├── apps/ # Business applications │ ├── mcp10x.yaml @@ -135,6 +154,15 @@ launchpad/ │ ├── mcp10x-credentials-sealed.yaml │ └── musicman-credentials.yaml │ +├── scripts/ # Operational helper scripts +│ ├── gitea-backup.sh # S3 backup helper (list/download) +│ ├── gitea-restore.sh +│ └── backup/ # Per-cloud backup reference scripts +│ ├── s3-minio.sh # S3-compatible (UpCloud, MinIO, Wasabi) +│ ├── aws-s3.sh # Native AWS S3 +│ ├── azure-blob.sh # Azure Blob Storage +│ └── gcp-gcs.sh # GCP Cloud Storage +│ ├── private/ # Local-only (Git-ignored) │ ├── *.yaml │ └── *.sh @@ -1621,6 +1649,79 @@ POST /loki/api/v1/push --- +## Cloud Overlay Pattern + +### Overview + +Cloud-specific configuration (StorageClass, LoadBalancer annotations, pricing models, etc.) lives in per-cloud overlay value files, **not** in `base/`. This means adding a new cloud provider (AKS, EKS, GKE) only requires a new overlay directory — no base changes. + +### How It Works + +Each ArgoCD Application uses **multi-source Helm values** with two value files: + +```yaml +# infra/base/gitea.yaml (example) +helm: + valueFiles: + - $values/infra/values/base/gitea-values.yaml # [0] cloud-agnostic + - $values/infra/values/upc-dev/gitea-values.yaml # [1] cloud-specific (default: upc-dev) +``` + +The `upc-prod` Kustomize overlay patches index `[1]` to swap the cloud-specific file: + +```yaml +# infra/overlays/upc-prod/kustomization.yaml +- target: + kind: Application + name: gitea + patch: | + - op: replace + path: /spec/sources/0/helm/valueFiles/1 + value: $values/infra/values/upc-prod/gitea-values.yaml +``` + +### Components Using Cloud Overlays + +| Component | Cloud-specific config | Overlay value file | +|-----------|----------------------|-------------------| +| **Traefik** | LB annotations, proxy protocol IPs | `traefik-values.yaml` | +| **Keycloak** | Hostname, TLS settings | `keycloak-values.yaml` | +| **Grafana** | Hostname, datasource URLs | `grafana-values.yaml` | +| **Gitea** | StorageClass (persistence + PostgreSQL) | `gitea-values.yaml` | +| **OpenCost** | Custom pricing model (CPU/RAM/storage rates) | `opencost-values.yaml` | + +### Backup CronJob + +The `gitea-backup` CronJob uses a generic `s3` alias for `minio/mc`. The actual endpoint and credentials come from the `gitea-backup-s3` Sealed Secret, which is per-cloud. Reference scripts for different cloud providers are in `scripts/backup/`: + +| Script | Provider | Tool | +|--------|----------|------| +| `s3-minio.sh` | S3-compatible (UpCloud, MinIO, Wasabi) | `minio/mc` | +| `aws-s3.sh` | AWS S3 | `aws` CLI | +| `azure-blob.sh` | Azure Blob Storage | `az` CLI | +| `gcp-gcs.sh` | GCP Cloud Storage | `gsutil` | + +### Adding a New Cloud Provider + +To add support for a new cloud (e.g., `aks-dev`): + +1. **Create overlay value directory**: `infra/values/aks-dev/` +2. **Add cloud-specific value files** for each component that needs one: + - `traefik-values.yaml` — LB annotations, proxy protocol config + - `keycloak-values.yaml` — hostname/TLS if different + - `grafana-values.yaml` — hostname/datasources if different + - `gitea-values.yaml` — `storageClass` for persistence + PostgreSQL + - `opencost-values.yaml` — `customPricing` cost model for your cloud +3. **Create a Kustomize overlay** (if needed): `infra/overlays/aks-prod/kustomization.yaml` + - Patch each Application's `valueFiles[1]` to point to `aks-prod/` files +4. **Create a root Application**: `_app-of-apps-aks-dev.yaml` pointing to the overlay +5. **Create Sealed Secrets** for the new cluster: + - `secrets/aks-dev/` — TLS certs, credentials, backup S3 config +6. **Update `gitea-backup-s3` secret** with the new cloud's S3-compatible endpoint +7. **Bootstrap**: `kubectl apply -f _app-of-apps-aks-dev.yaml -n argocd` + +--- + ## Glossary ### Terms diff --git a/infra/base/gitea.yaml b/infra/base/gitea.yaml index ba806f5..cc4f60f 100644 --- a/infra/base/gitea.yaml +++ b/infra/base/gitea.yaml @@ -22,6 +22,7 @@ spec: releaseName: gitea valueFiles: - $values/infra/values/base/gitea-values.yaml + - $values/infra/values/upc-dev/gitea-values.yaml - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git targetRevision: HEAD diff --git a/infra/base/opencost.yaml b/infra/base/opencost.yaml index 6984f3b..a102906 100644 --- a/infra/base/opencost.yaml +++ b/infra/base/opencost.yaml @@ -22,6 +22,7 @@ spec: releaseName: opencost valueFiles: - $values/infra/values/base/opencost-values.yaml + - $values/infra/values/upc-dev/opencost-values.yaml - repoURL: git@github.com:fortedigital/sturdy-adventure.git targetRevision: HEAD diff --git a/infra/overlays/upc-prod/kustomization.yaml b/infra/overlays/upc-prod/kustomization.yaml index ebfc179..9242d39 100644 --- a/infra/overlays/upc-prod/kustomization.yaml +++ b/infra/overlays/upc-prod/kustomization.yaml @@ -31,6 +31,24 @@ patches: path: /spec/sources/0/helm/valueFiles/1 value: $values/infra/values/upc-prod/grafana-values.yaml +# Gitea: swap upc-dev → upc-prod +- target: + kind: Application + name: gitea + patch: | + - op: replace + path: /spec/sources/0/helm/valueFiles/1 + value: $values/infra/values/upc-prod/gitea-values.yaml + +# OpenCost: swap upc-dev → upc-prod +- target: + kind: Application + name: opencost + patch: | + - op: replace + path: /spec/sources/0/helm/valueFiles/1 + value: $values/infra/values/upc-prod/opencost-values.yaml + # Secrets: change path to upc-prod - target: kind: Application diff --git a/infra/values/base/gitea-values.yaml b/infra/values/base/gitea-values.yaml index 2153b89..635ae37 100644 --- a/infra/values/base/gitea-values.yaml +++ b/infra/values/base/gitea-values.yaml @@ -130,7 +130,6 @@ persistence: size: 10Gi accessModes: - ReadWriteOnce - storageClass: upcloud-block-storage-maxiops # -- Recreate strategy to avoid Multi-Attach errors with RWO volumes strategy: @@ -156,7 +155,6 @@ postgresql: persistence: enabled: true size: 8Gi - storageClass: upcloud-block-storage-maxiops resources: requests: cpu: 100m diff --git a/infra/values/base/opencost-values.yaml b/infra/values/base/opencost-values.yaml index 39d73cc..1b97209 100644 --- a/infra/values/base/opencost-values.yaml +++ b/infra/values/base/opencost-values.yaml @@ -10,18 +10,6 @@ opencost: serviceName: prometheus-server namespaceName: monitoring port: 80 - customPricing: - enabled: true - provider: custom - costModel: - description: "UpCloud 4-node cluster pricing" - CPU: "5.86" - RAM: "1.46" - GPU: "0" - storage: "0.34" - zoneNetworkEgress: "0" - regionNetworkEgress: "0" - internetNetworkEgress: "0" ui: enabled: false service: diff --git a/infra/values/upc-dev/gitea-values.yaml b/infra/values/upc-dev/gitea-values.yaml new file mode 100644 index 0000000..151047f --- /dev/null +++ b/infra/values/upc-dev/gitea-values.yaml @@ -0,0 +1,7 @@ +# UpCloud-specific: block storage class for Gitea + PostgreSQL +persistence: + storageClass: upcloud-block-storage-maxiops +postgresql: + primary: + persistence: + storageClass: upcloud-block-storage-maxiops diff --git a/infra/values/upc-dev/opencost-values.yaml b/infra/values/upc-dev/opencost-values.yaml new file mode 100644 index 0000000..51fd0a4 --- /dev/null +++ b/infra/values/upc-dev/opencost-values.yaml @@ -0,0 +1,15 @@ +# UpCloud-specific: custom pricing model +opencost: + exporter: + customPricing: + enabled: true + provider: custom + costModel: + description: "UpCloud 4-node cluster pricing" + CPU: "5.86" + RAM: "1.46" + GPU: "0" + storage: "0.34" + zoneNetworkEgress: "0" + regionNetworkEgress: "0" + internetNetworkEgress: "0" diff --git a/infra/values/upc-prod/gitea-values.yaml b/infra/values/upc-prod/gitea-values.yaml new file mode 100644 index 0000000..151047f --- /dev/null +++ b/infra/values/upc-prod/gitea-values.yaml @@ -0,0 +1,7 @@ +# UpCloud-specific: block storage class for Gitea + PostgreSQL +persistence: + storageClass: upcloud-block-storage-maxiops +postgresql: + primary: + persistence: + storageClass: upcloud-block-storage-maxiops diff --git a/infra/values/upc-prod/opencost-values.yaml b/infra/values/upc-prod/opencost-values.yaml new file mode 100644 index 0000000..51fd0a4 --- /dev/null +++ b/infra/values/upc-prod/opencost-values.yaml @@ -0,0 +1,15 @@ +# UpCloud-specific: custom pricing model +opencost: + exporter: + customPricing: + enabled: true + provider: custom + costModel: + description: "UpCloud 4-node cluster pricing" + CPU: "5.86" + RAM: "1.46" + GPU: "0" + storage: "0.34" + zoneNetworkEgress: "0" + regionNetworkEgress: "0" + internetNetworkEgress: "0" diff --git a/scripts/backup/aws-s3.sh b/scripts/backup/aws-s3.sh new file mode 100644 index 0000000..679245b --- /dev/null +++ b/scripts/backup/aws-s3.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +# AWS S3 backup upload (native AWS CLI) +# Uses: aws cli v2 +# Env: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION, S3_BUCKET + +BACKUP_FILE="${1:?Usage: $0 }" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +KEY="gitea-dump-${TIMESTAMP}.zip" + +echo "Uploading ${KEY}..." +aws s3 cp "$BACKUP_FILE" "s3://${S3_BUCKET}/${KEY}" +echo "Upload complete." + +# Prune backups older than 7 days +echo "Pruning backups older than 7 days..." +CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%S) +aws s3api list-objects-v2 --bucket "${S3_BUCKET}" --query "Contents[?LastModified<'${CUTOFF}'].Key" --output text \ + | tr '\t' '\n' \ + | while read -r key; do + [ -n "$key" ] && aws s3 rm "s3://${S3_BUCKET}/${key}" && echo "Deleted: ${key}" + done +echo "Pruning complete." diff --git a/scripts/backup/azure-blob.sh b/scripts/backup/azure-blob.sh new file mode 100644 index 0000000..bb095ab --- /dev/null +++ b/scripts/backup/azure-blob.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail +# Azure Blob Storage backup upload +# Uses: az cli +# Env: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, AZURE_CONTAINER + +BACKUP_FILE="${1:?Usage: $0 }" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +KEY="gitea-dump-${TIMESTAMP}.zip" + +echo "Uploading ${KEY}..." +az storage blob upload \ + --account-name "${AZURE_STORAGE_ACCOUNT}" \ + --account-key "${AZURE_STORAGE_KEY}" \ + --container-name "${AZURE_CONTAINER}" \ + --name "${KEY}" \ + --file "$BACKUP_FILE" \ + --overwrite +echo "Upload complete." + +# Prune backups older than 7 days +echo "Pruning backups older than 7 days..." +CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%SZ) +az storage blob list \ + --account-name "${AZURE_STORAGE_ACCOUNT}" \ + --account-key "${AZURE_STORAGE_KEY}" \ + --container-name "${AZURE_CONTAINER}" \ + --query "[?properties.lastModified<'${CUTOFF}'].name" -o tsv \ + | while read -r name; do + [ -n "$name" ] && az storage blob delete \ + --account-name "${AZURE_STORAGE_ACCOUNT}" \ + --account-key "${AZURE_STORAGE_KEY}" \ + --container-name "${AZURE_CONTAINER}" \ + --name "$name" && echo "Deleted: ${name}" + done +echo "Pruning complete." diff --git a/scripts/backup/gcp-gcs.sh b/scripts/backup/gcp-gcs.sh new file mode 100644 index 0000000..7e9ed60 --- /dev/null +++ b/scripts/backup/gcp-gcs.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +# GCP Cloud Storage backup upload +# Uses: gsutil (gcloud SDK) +# Env: GCS_BUCKET (e.g. gs://my-bucket) + +BACKUP_FILE="${1:?Usage: $0 }" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +KEY="gitea-dump-${TIMESTAMP}.zip" + +echo "Uploading ${KEY}..." +gsutil cp "$BACKUP_FILE" "${GCS_BUCKET}/${KEY}" +echo "Upload complete." + +# Prune backups older than 7 days — GCS lifecycle rules are preferred, +# but this works as a manual fallback +echo "Pruning backups older than 7 days..." +CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%SZ) +gsutil ls -l "${GCS_BUCKET}/" \ + | grep 'gitea-dump-' \ + | while read -r size date name; do + if [[ "$date" < "$CUTOFF" ]]; then + gsutil rm "$name" && echo "Deleted: ${name}" + fi + done +echo "Pruning complete." diff --git a/scripts/backup/s3-minio.sh b/scripts/backup/s3-minio.sh new file mode 100644 index 0000000..fb73537 --- /dev/null +++ b/scripts/backup/s3-minio.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +# S3-compatible backup upload (UpCloud Objects, MinIO, Wasabi, etc.) +# Uses: minio/mc +# Env: S3_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET + +BACKUP_FILE="${1:?Usage: $0 }" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) +KEY="gitea-dump-${TIMESTAMP}.zip" + +mc alias set s3 "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}" + +echo "Uploading ${KEY}..." +mc cp "$BACKUP_FILE" "s3/${S3_BUCKET}/${KEY}" +echo "Upload complete." + +# Prune backups older than 7 days +echo "Pruning backups older than 7 days..." +mc rm --older-than 7d --force "s3/${S3_BUCKET}/" 2>&1 || true +echo "Pruning complete." diff --git a/scripts/gitea-backup.sh b/scripts/gitea-backup.sh index 397a6fa..9d6cdb0 100644 --- a/scripts/gitea-backup.sh +++ b/scripts/gitea-backup.sh @@ -13,7 +13,7 @@ NAMESPACE="gitea" SECRET="gitea-backup-s3" IMAGE="minio/mc:latest" POD_NAME="gitea-backup-helper" -ALIAS_CMD='mc alias set upcloud ${S3_ENDPOINT} ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} > /dev/null' +ALIAS_CMD='mc alias set s3 ${S3_ENDPOINT} ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} > /dev/null' cleanup() { kubectl -n "$NAMESPACE" delete pod "$POD_NAME" --ignore-not-found --grace-period=0 > /dev/null 2>&1 || true @@ -41,7 +41,7 @@ mc_run() { case "${1:-help}" in list) echo "Listing backups..." - mc_run 'mc ls upcloud/${S3_BUCKET}/' + mc_run 'mc ls s3/${S3_BUCKET}/' ;; download) @@ -49,7 +49,7 @@ case "${1:-help}" in if [ "$FILE" = "latest" ]; then echo "Finding latest backup..." - FILE=$(mc_run 'mc ls upcloud/${S3_BUCKET}/' | sort | tail -1 | awk '{print $NF}' | tr -d '[:space:]') + FILE=$(mc_run 'mc ls s3/${S3_BUCKET}/' | sort | tail -1 | awk '{print $NF}' | tr -d '[:space:]') if [ -z "$FILE" ]; then echo "No backups found." exit 1 @@ -74,7 +74,7 @@ case "${1:-help}" in kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$POD_NAME" --timeout=60s > /dev/null 2>&1 echo "Saving to ./$FILE ..." - kubectl -n "$NAMESPACE" exec "$POD_NAME" -- sh -c "${ALIAS_CMD} && mc cat upcloud/\${S3_BUCKET}/$FILE" > "./$FILE" + kubectl -n "$NAMESPACE" exec "$POD_NAME" -- sh -c "${ALIAS_CMD} && mc cat s3/\${S3_BUCKET}/$FILE" > "./$FILE" cleanup echo "Downloaded: ./$FILE"