feat(infra): drop in-cluster minio, add pg backup + PVC protection

PROD: object storage moves to UpCloud Managed Object Storage (existing
drops bucket) instead of single-node in-cluster MinIO — durable,
UpCloud-replicated, no PVC to back up.

- Remove forte-drop-minio StatefulSet entirely.
- Add forte-drop-pg-backup CronJob: nightly pg_dump -> gzip -> upload to
  s3://drops/_pgbackups/ (collision-proof prefix), 30-day retention.
  Reuses forte-drop-secrets S3 creds (app user has s3:* on drops).
- PVC prune/delete protection on the postgres volumeClaimTemplate.

infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml

@@ -1,40 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: forte-drop-minio
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-  labels:
-    app.kubernetes.io/name: forte-drop-minio
-    app.kubernetes.io/part-of: apps
-    app.kubernetes.io/managed-by: argocd
-  finalizers:
-  - resources-finalizer.argocd.argoproj.io
-spec:
-  project: default
-
-  source:
-    repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
-    targetRevision: HEAD
-    path: infra/overlays/upc-dev/forte-drop-minio/resources
-
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: forte-drop
-
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-      allowEmpty: false
-    syncOptions:
-    - CreateNamespace=true
-    - Validate=true
-    - ServerSideApply=true
-
-  ignoreDifferences:
-  - group: apps
-    kind: StatefulSet
-    jsonPointers:
-    - /spec/volumeClaimTemplates

infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml

@@ -1,4 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-- forte-drop-minio.yaml

infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml

@@ -1,14 +0,0 @@
----
-apiVersion: bitnami.com/v1alpha1
-kind: SealedSecret
-metadata:
-  name: forte-drop-minio-creds
-  namespace: forte-drop
-spec:
-  encryptedData:
-    root-password: AgCaEHfMk/fClLDpvNvAQKIFPHRkZZVqw7/Ct4GO7v0kVpdMbQiisJ+dkYlpFsOCflQ7HaruF8Cf8OiGX5qlfUx5R4WG+msVvtgGd9zK3DJ2w0DiD5aIlyZQkAspbsUPJSUmfPGb1wrKIlCLd3vOPZJcGr1/nH9Dr2VWJJuEDvootJHvXzq3PZlJ9aGDglvyOWsibHUKQdFIhdg1Ajs3kGYIZYmVsAIyqjSXeo1jsB9nUiFy2ufKO5STiF6KQ+EPOjv+tyfhBB6Tdu46d82BaFY3CML/HV2lbkPlsCkByIb16g+Sbv4yTaSB/VVSLeqD2MyD9J4QRB5wd2luEFjpopW0tXUCqks/CoFX25gTd6TF0joaYXFuMI4Pf9EgBpXY0nZGW+K+Kqo8sHaSWZR6CT1wubHzky1GN8+8GTj1nGvqRSgVOIyCJqP+CylAuhEzYgGs9OCwX9iulBlQjQnpP6dsPhp/QB4C9VH4xFU2kc1cbKWiGFMpyXxMxBFGuEKjz+KK577HlwxqITcaMhBxvIS/PimOrAL40vl65dDtjUaHFez4Hv6u13YO+F21m5J9ChFr+l4X8i96GPoZQ4fshOCmjkZ/eNHZTMLA0iViD7Zqqj1l4rgd3PW+A9Sd1kq1aeRPoyF3FHBi17DulR+2qgrdO0QrLnKVRMRDmxaNvleyLJVX3NwJn2yjZrlfQPqzCxTmAzBTnUMCwK5WOtzVQpH9TP8xC4yj1sxhFBv1cmH5Jg==
-    root-user: AgDVJTL+2vSqkv6jMERD4TPUMA9/41WA0+9tpW0Had3RpkvDUAe2dCTcIHeLFra0XMxZE7z9TbJqP1d0rDsk5SSoSqRq+wR1gizjMuiBRSt8icVgClHvNxwXfoGz4Z0FH6IR9M5G+o36hy5kkprJcv1OJ4u+ddXs2sv3zEGUaS6Am8jscij2IJYSDK3jExpv/UEfKKqWcHrfbcoIILdk3t+FGjX9SiTC6s7DEBgbyaVjQavvA+q4GasLm2vTaFPE+G6rb10LB3Yq4pF8h7AplBZwIylEeuaVE1pivLkNHMqTSZbNaV3DZRUGTa4EwQoM8dC1CKuhBONNt25SfQ3n/Aiv6gl1eKCSEcoq4zUDHGsa4iTyqT5VyoKI6VZ0reEJVaut/4wavuc/dNkBHiJ9Y1IZwjpYLsTy+zrnX2vv1qYLBKg59AlEg73lB0HcLFkQBVvR+dkE1pKHFDrR5BDww6RykCF4/1eGS23MuPr0SudEejYmm/P/5FrBMQ1mB8ZhPw074brxkLtgW4qgsLlSN0GkFcqbR7b4pI4SPMmXBO+p9iX6dI8BgA8ZoL84jrmHlUMXeCnHiN5RV8nNt1b0DO7LK0cyHe1YFH4vHpIDXv9K1wxLuwLfjk4xCP/e6wlximdPzaKx6uBCW8xRs0BCnoLsWP7uXnF+R/ELQlyHwsrVyMXnODewhsrcqnqORm8kOjxdR8w2hkqCFqYj3c6h5nkA
-  template:
-    metadata:
-      name: forte-drop-minio-creds
-      namespace: forte-drop

infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-- minio.yaml
-- forte-drop-minio-creds-sealed.yaml

infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml

@@ -1,146 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: forte-drop-minio
-  namespace: forte-drop
-  labels:
-    app.kubernetes.io/name: minio
-    app.kubernetes.io/instance: forte-drop
-    app.kubernetes.io/component: object-storage
-spec:
-  type: ClusterIP
-  ports:
-  - name: http-api
-    port: 9000
-    targetPort: http-api
-  - name: http-console
-    port: 9001
-    targetPort: http-console
-  selector:
-    app.kubernetes.io/name: minio
-    app.kubernetes.io/instance: forte-drop
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: forte-drop-minio
-  namespace: forte-drop
-  labels:
-    app.kubernetes.io/name: minio
-    app.kubernetes.io/instance: forte-drop
-    app.kubernetes.io/component: object-storage
-spec:
-  serviceName: forte-drop-minio
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: minio
-      app.kubernetes.io/instance: forte-drop
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: minio
-        app.kubernetes.io/instance: forte-drop
-        app.kubernetes.io/component: object-storage
-    spec:
-      containers:
-      - name: minio
-        image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z
-        args:
-        - server
-        - /data
-        - --console-address
-        - :9001
-        ports:
-        - name: http-api
-          containerPort: 9000
-        - name: http-console
-          containerPort: 9001
-        env:
-        - name: MINIO_ROOT_USER
-          valueFrom:
-            secretKeyRef:
-              name: forte-drop-minio-creds
-              key: root-user
-        - name: MINIO_ROOT_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: forte-drop-minio-creds
-              key: root-password
-        volumeMounts:
-        - name: data
-          mountPath: /data
-        livenessProbe:
-          httpGet:
-            path: /minio/health/live
-            port: http-api
-          initialDelaySeconds: 30
-          periodSeconds: 20
-        readinessProbe:
-          httpGet:
-            path: /minio/health/ready
-            port: http-api
-          initialDelaySeconds: 5
-          periodSeconds: 5
-        resources:
-          requests:
-            cpu: 100m
-            memory: 256Mi
-          limits:
-            cpu: 500m
-            memory: 1Gi
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes:
-      - ReadWriteOnce
-      storageClassName: upcloud-block-storage-maxiops
-      resources:
-        requests:
-          storage: 20Gi
----
-# Bootstrap job — creates the 'drops' bucket once MinIO is reachable.
-# Idempotent: `mc mb --ignore-existing` skips if bucket already exists.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: forte-drop-minio-bootstrap
-  namespace: forte-drop
-  labels:
-    app.kubernetes.io/name: minio
-    app.kubernetes.io/instance: forte-drop
-    app.kubernetes.io/component: bootstrap
-  annotations:
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: HookSucceeded
-spec:
-  backoffLimit: 5
-  template:
-    spec:
-      restartPolicy: OnFailure
-      containers:
-      - name: mc
-        image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z
-        env:
-        - name: MINIO_ROOT_USER
-          valueFrom:
-            secretKeyRef:
-              name: forte-drop-minio-creds
-              key: root-user
-        - name: MINIO_ROOT_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: forte-drop-minio-creds
-              key: root-password
-        command:
-        - sh
-        - -c
-        - |
-          set -euo pipefail
-          until mc alias set local http://forte-drop-minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do
-            echo "waiting for minio..."
-            sleep 2
-          done
-          mc mb --ignore-existing local/drops
-          echo "bucket 'drops' ready"

infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml

@@ -3,3 +3,4 @@ kind: Kustomization
 resources:
 - postgresql.yaml
 - forte-drop-pg-creds-sealed.yaml
+- pg-backup-cronjob.yaml

infra/overlays/upc-dev/forte-drop-postgresql/resources/pg-backup-cronjob.yaml

@@ -0,0 +1,93 @@
+# Nightly logical backup of the forte-drop Postgres → UpCloud Managed Object Storage.
+# Dumps to s3://drops/_pgbackups/ (the `_` prefix is collision-proof: app slugs match
+# /^[a-z0-9][a-z0-9-]{0,62}$/ and can never start with `_`). Retains 30 days.
+#
+# Pod shape: initContainer pg_dump → shared emptyDir → mc upload + retention prune.
+# Both images pinned. S3 creds reuse forte-drop-secrets (the app's UpCloud user has
+# s3:* on the drops bucket). PG creds from forte-drop-pg-creds.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: forte-drop-pg-backup
+  namespace: forte-drop
+  labels:
+    app.kubernetes.io/name: postgresql
+    app.kubernetes.io/instance: forte-drop
+    app.kubernetes.io/component: backup
+spec:
+  schedule: "0 2 * * *"           # 02:00 UTC daily
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: postgresql
+            app.kubernetes.io/instance: forte-drop
+            app.kubernetes.io/component: backup
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            fsGroup: 65532
+          volumes:
+          - name: work
+            emptyDir: {}
+          initContainers:
+          - name: dump
+            image: postgres:16-alpine
+            command:
+            - sh
+            - -c
+            - |
+              set -euo pipefail
+              TS=$(date -u +%Y%m%dT%H%M%SZ)
+              echo "dumping to /work/forte-drop-${TS}.sql.gz"
+              PGPASSWORD="$PGPASSWORD" pg_dump \
+                -h forte-drop-postgresql.forte-drop.svc \
+                -p 5432 -U "$PGUSER" -d drops \
+                --no-owner --no-privileges \
+                | gzip -9 > "/work/forte-drop-${TS}.sql.gz"
+              echo "dump complete: $(ls -lh /work/)"
+            env:
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef: { name: forte-drop-pg-creds, key: pgusername }
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef: { name: forte-drop-pg-creds, key: pgpassword }
+            volumeMounts:
+            - name: work
+              mountPath: /work
+          containers:
+          - name: upload
+            image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z
+            command:
+            - sh
+            - -c
+            - |
+              set -euo pipefail
+              mc alias set obj "$S3_ENDPOINT" "$S3_KEY" "$S3_SECRET"
+              mc cp /work/*.sql.gz "obj/${S3_BUCKET}/_pgbackups/"
+              echo "uploaded. pruning backups older than 30d:"
+              mc rm --recursive --force --older-than 30d "obj/${S3_BUCKET}/_pgbackups/" || true
+              echo "backup retention pass complete"
+            env:
+            - name: S3_ENDPOINT
+              valueFrom:
+                secretKeyRef: { name: forte-drop-secrets, key: S3_ENDPOINT }
+            - name: S3_BUCKET
+              value: "drops"
+            - name: S3_KEY
+              valueFrom:
+                secretKeyRef: { name: forte-drop-secrets, key: S3_KEY }
+            - name: S3_SECRET
+              valueFrom:
+                secretKeyRef: { name: forte-drop-secrets, key: S3_SECRET }
+            volumeMounts:
+            - name: work
+              mountPath: /work

infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml

@@ -94,6 +94,8 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: data
+      annotations:
+        argocd.argoproj.io/sync-options: Prune=false,Delete=false
     spec:
       accessModes:
       - ReadWriteOnce

infra/overlays/upc-dev/kustomization.yaml

@@ -5,7 +5,6 @@ resources:
 - vaultwarden-postgresql
 - vaultwarden
 - forte-drop-postgresql
-- forte-drop-minio
 
 # No patches needed — base already has "upc-dev" paths
 # upc-dev is the default/base cluster