docs(infra): pg backup & restore runbook for forte-drop

Covers: nightly backup mechanism, listing backups, manual trigger,
full restore procedure (2-pod mc-download + psql-pipe), verification,
object-data note, and a disaster-scenario recovery table.

cluster-resources/letsencrypt-issuer.yaml

@@ -12,18 +12,6 @@ spec:
     privateKeySecretRef:
       name: letsencrypt-staging-key
     solvers:
-    # DNS-01 solver for wildcard certificates (*.example.com)
-    - dns01:
-        cloudflare:
-          email: danijels@gmail.com
-          apiTokenSecretRef:
-            name: cloudflare-api-token-secret
-            key: api-token
-      selector:
-        dnsNames:
-        - '*.example.com'
-        - 'example.com'
-    # HTTP-01 fallback for non-wildcard certificates
     - http01:
         ingress:
           class: traefik
@@ -42,116 +30,6 @@ spec:
     privateKeySecretRef:
       name: letsencrypt-prod-key
     solvers:
-    # DNS-01 solver for wildcard certificates (*.example.com)
-    - dns01:
-        cloudflare:
-          email: danijels@gmail.com
-          apiTokenSecretRef:
-            name: cloudflare-api-token-secret
-            key: api-token
-      selector:
-        dnsNames:
-        - '*.example.com'
-        - 'example.com'
-    # HTTP-01 fallback for non-wildcard certificates
     - http01:
         ingress:
           class: traefik
-
-# =============================================================================
-# DNS PROVIDER EXAMPLES - Uncomment and configure based on your provider:
-# =============================================================================
-
-# -----------------------------------------------------------------------------
-# Option 1: Cloudflare (recommended - supports API tokens with limited scope)
-# -----------------------------------------------------------------------------
-# Create secret with: kubectl create secret generic cloudflare-api-token-secret \
-#   --from-literal=api-token=YOUR_CLOUDFLARE_API_TOKEN -n cert-manager
-#
-# dns01:
-#   cloudflare:
-#     email: your-cloudflare-email@example.com
-#     apiTokenSecretRef:
-#       name: cloudflare-api-token-secret
-#       key: api-token
-
-# -----------------------------------------------------------------------------
-# Option 2: AWS Route53
-# -----------------------------------------------------------------------------
-# Create secret with: kubectl create secret generic route53-credentials \
-#   --from-literal=secret-access-key=YOUR_SECRET_KEY -n cert-manager
-#
-# dns01:
-#   route53:
-#     region: us-east-1
-#     hostedZoneID: ZXXXXXXXXXXXXX  # Optional: auto-detected if not specified
-#     accessKeyID: YOUR_ACCESS_KEY_ID
-#     secretAccessKeySecretRef:
-#       name: route53-credentials
-#       key: secret-access-key
-
-# -----------------------------------------------------------------------------
-# Option 3: Azure DNS
-# -----------------------------------------------------------------------------
-# Create secret with: kubectl create secret generic azuredns-config \
-#   --from-literal=client-secret=YOUR_CLIENT_SECRET -n cert-manager
-#
-# dns01:
-#   azureDNS:
-#     subscriptionID: YOUR_SUBSCRIPTION_ID
-#     resourceGroupName: YOUR_RESOURCE_GROUP
-#     hostedZoneName: example.com
-#     environment: AzurePublicCloud
-#     managedIdentity:
-#       clientID: YOUR_MANAGED_IDENTITY_CLIENT_ID  # For AKS with pod identity
-#     # OR use service principal:
-#     # clientID: YOUR_SERVICE_PRINCIPAL_CLIENT_ID
-#     # clientSecretSecretRef:
-#     #   name: azuredns-config
-#     #   key: client-secret
-
-# -----------------------------------------------------------------------------
-# Option 4: Google Cloud DNS
-# -----------------------------------------------------------------------------
-# Create secret with service account JSON key:
-# kubectl create secret generic clouddns-service-account \
-#   --from-file=service-account.json=path/to/key.json -n cert-manager
-#
-# dns01:
-#   cloudDNS:
-#     project: YOUR_GCP_PROJECT_ID
-#     hostedZoneName: example-com  # Managed zone name in Cloud DNS
-#     serviceAccountSecretRef:
-#       name: clouddns-service-account
-#       key: service-account.json
-
-# -----------------------------------------------------------------------------
-# Option 5: GoDaddy
-# -----------------------------------------------------------------------------
-# Requires external webhook: https://github.com/snowdrop/godaddy-webhook
-#
-# dns01:
-#   webhook:
-#     groupName: acme.yourcompany.com
-#     solverName: godaddy
-#     config:
-#       apiKeySecretRef:
-#         name: godaddy-api-credentials
-#         key: api-key
-#       apiSecretSecretRef:
-#         name: godaddy-api-credentials
-#         key: api-secret
-
-# -----------------------------------------------------------------------------
-# Option 6: Manual/Dynamic DNS (for homelab)
-# -----------------------------------------------------------------------------
-# Requires RFC2136 provider or external webhook
-#
-# dns01:
-#   rfc2136:
-#     nameserver: your-dns-server.example.com
-#     tsigKeyName: cert-manager-key
-#     tsigAlgorithm: HMACSHA256
-#     tsigSecretSecretRef:
-#       name: tsig-secret
-#       key: secret

cluster-resources/wildcard-certificate-example.yaml

@@ -1,92 +0,0 @@
----
-# Example: Wildcard Certificate for *.example.com
-# This creates a certificate that covers ALL subdomains of example.com
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: wildcard-example-com
-  namespace: default  # Change to your application's namespace
-spec:
-  # The secret where the TLS certificate will be stored
-  secretName: wildcard-example-com-tls
-  
-  # Use the production issuer (use letsencrypt-staging for testing)
-  issuerRef:
-    name: letsencrypt-prod
-    kind: ClusterIssuer
-    
-  # DNS names this certificate will cover
-  # Both wildcard AND apex domain are recommended
-  dnsNames:
-    - '*.example.com'    # Covers: app.example.com, api.example.com, etc.
-    - 'example.com'      # Also include apex domain explicitly
-  
-  # Optional: Configure certificate duration and renewal
-  duration: 2160h0m0s    # 90 days (Let's Encrypt default)
-  renewBefore: 720h0m0s  # Renew 30 days before expiry
-  
-  # Optional: Private key settings
-  privateKey:
-    algorithm: RSA
-    encoding: PKCS1
-    size: 4096
-
----
-# Example: Using the wildcard certificate with a Traefik IngressRoute
-apiVersion: traefik.containo.us/v1alpha1
-kind: IngressRoute
-metadata:
-  name: app-ingress
-  namespace: default
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    # Match any subdomain - the wildcard cert covers all of them
-    - match: Host(`app.example.com`) || Host(`api.example.com`) || Host(`www.example.com`)
-      kind: Rule
-      services:
-        - name: my-service
-          port: 80
-  tls:
-    # Reference the secret created by the Certificate
-    secretName: wildcard-example-com-tls
-
----
-# Example: Using wildcard certificate with standard Kubernetes Ingress
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: wildcard-ingress
-  namespace: default
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    traefik.ingress.kubernetes.io/router.entrypoints: websecure
-    traefik.ingress.kubernetes.io/router.tls: "true"
-spec:
-  tls:
-    - hosts:
-        - '*.example.com'
-        - 'example.com'
-      secretName: wildcard-example-com-tls
-  rules:
-    - host: app.example.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: app-service
-                port:
-                  number: 80
-    - host: api.example.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: api-service
-                port:
-                  number: 80

infra/overlays/upc-dev/forte-drop-postgresql/RESTORE.md

@@ -0,0 +1,143 @@
+# forte-drop Postgres — backup & restore runbook
+
+## What gets backed up
+
+A CronJob (`forte-drop-pg-backup`, namespace `forte-drop`) runs nightly at **02:00 UTC**:
+
+1. `pg_dump` of the `drops` database → gzip.
+2. Upload to **UpCloud Managed Object Storage**: `s3://drops/_pgbackups/forte-drop-<TS>.sql.gz`
+   (the `_pgbackups/` prefix is collision-proof: app slugs match `/^[a-z0-9][a-z0-9-]{0,62}$/`
+   and can never start with `_`).
+3. Retention: dumps older than **30 days** are pruned.
+
+S3 creds come from the `forte-drop-secrets` Secret (`S3_ENDPOINT` / `S3_KEY` / `S3_SECRET`).
+Postgres creds from `forte-drop-pg-creds` (`pgusername` / `pgpassword`).
+
+> **Object storage is the durable tier.** App data + DB backups both live in UpCloud
+> Managed Object Storage (replicated by UpCloud). The in-cluster Postgres PVC is the
+> live working copy; the nightly dump is the recovery point. The PVC carries
+> `Prune=false,Delete=false` so ArgoCD never deletes it.
+
+## Prerequisites
+
+```bash
+export KUBECONFIG=~/Downloads/dev-fd-no-svg1_kubeconfig.yaml
+# Confirm the namespace + DB pod are up:
+kubectl -n forte-drop get pods -l app.kubernetes.io/name=postgresql
+```
+
+## List available backups
+
+```bash
+# Run an ephemeral mc pod with the app's S3 creds:
+kubectl -n forte-drop run mc-list --rm -it --restart=Never \
+  --image=quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z \
+  --overrides='{"spec":{"containers":[{"name":"mc","image":"quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z","command":["sh","-c","mc alias set obj \"$S3_ENDPOINT\" \"$S3_KEY\" \"$S3_SECRET\" >/dev/null && mc ls obj/drops/_pgbackups/"],"envFrom":[{"secretRef":{"name":"forte-drop-secrets"}}]}]}}'
+```
+
+## Manually trigger a backup (before risky changes)
+
+```bash
+kubectl -n forte-drop create job --from=cronjob/forte-drop-pg-backup pg-backup-manual-$(date +%s)
+# Watch:
+kubectl -n forte-drop get jobs -l app.kubernetes.io/component=backup
+kubectl -n forte-drop logs -l app.kubernetes.io/component=backup --tail=40
+```
+
+## Restore a dump
+
+> **Destructive.** This overwrites the live `drops` database. Take a fresh manual
+> backup first (above) and confirm with whoever owns the data before proceeding.
+
+### 1. Pick the dump to restore
+
+List backups (above), choose `forte-drop-<TS>.sql.gz`.
+
+### 2. Run a restore pod that pulls the dump and pipes it into Postgres
+
+```bash
+DUMP="forte-drop-20260530T020000Z.sql.gz"   # <-- set to the chosen file
+
+kubectl -n forte-drop run pg-restore --rm -it --restart=Never \
+  --image=postgres:16-alpine \
+  --overrides='{
+    "spec": {
+      "containers": [{
+        "name": "restore",
+        "image": "postgres:16-alpine",
+        "command": ["sh","-c","set -euo pipefail; \
+          apk add --no-cache curl >/dev/null; \
+          # download via mc is simpler — use a 2-step instead (see note). \
+          echo placeholder"],
+        "envFrom": [
+          {"secretRef":{"name":"forte-drop-pg-creds"}},
+          {"secretRef":{"name":"forte-drop-secrets"}}
+        ]
+      }]
+    }
+  }'
+```
+
+**Simpler 2-pod approach (recommended — avoids cramming mc + psql in one image):**
+
+```bash
+DUMP="forte-drop-20260530T020000Z.sql.gz"
+
+# (a) Download the dump from object storage to a local file:
+kubectl -n forte-drop run mc-get --rm -it --restart=Never \
+  --image=quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z \
+  --overrides='{"spec":{"containers":[{"name":"mc","image":"quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z","command":["sh","-c","mc alias set obj \"$S3_ENDPOINT\" \"$S3_KEY\" \"$S3_SECRET\" >/dev/null && mc cat obj/drops/_pgbackups/'"$DUMP"'"],"envFrom":[{"secretRef":{"name":"forte-drop-secrets"}}]}]}}' \
+  > /tmp/$DUMP
+
+# (b) Pipe it into the live Postgres via the service:
+gunzip -c /tmp/$DUMP | kubectl -n forte-drop run pg-restore --rm -i --restart=Never \
+  --image=postgres:16-alpine \
+  --overrides='{"spec":{"containers":[{"name":"psql","image":"postgres:16-alpine","stdin":true,"command":["sh","-c","PGPASSWORD=\"$pgpassword\" psql -h forte-drop-postgresql.forte-drop.svc -U \"$pgusername\" -d drops"],"env":[{"name":"pgusername","valueFrom":{"secretKeyRef":{"name":"forte-drop-pg-creds","key":"pgusername"}}},{"name":"pgpassword","valueFrom":{"secretKeyRef":{"name":"forte-drop-pg-creds","key":"pgpassword"}}}]}]}}'
+```
+
+> The app's schema is created idempotently on boot (`CREATE TABLE IF NOT EXISTS` +
+> `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` in `src/repo/pg.ts`), and `pg_dump`
+> output includes the data. For a clean restore into a fresh DB this just works.
+> To restore over an existing DB with conflicting rows, drop/recreate the `drops`
+> database first (coordinate downtime — scale the web Deployment to 0 during the
+> restore so the app isn't writing).
+
+### 3. Verify
+
+```bash
+kubectl -n forte-drop run pg-check --rm -it --restart=Never \
+  --image=postgres:16-alpine \
+  --env="PGPASSWORD=$(kubectl -n forte-drop get secret forte-drop-pg-creds -o jsonpath='{.data.pgpassword}' | base64 -d)" \
+  --command -- psql -h forte-drop-postgresql.forte-drop.svc -U drops -d drops \
+  -c "SELECT count(*) AS drops FROM drops;" -c "SELECT count(*) AS view_hits FROM view_hits;"
+```
+
+### 4. Bring the app back
+
+```bash
+# If you scaled web to 0 for the restore:
+kubectl -n forte-drop scale deploy/forte-drop --replicas=2
+```
+
+## Object data (uploaded drop files)
+
+Drop files live in `s3://drops/<slug>/...` in the same managed bucket. They are
+**not** part of the pg backup (the dump only holds metadata). Object storage is
+UpCloud-managed/replicated, so no separate file backup is configured. If a
+file-level backup is later required, mirror the bucket to a second bucket/region:
+
+```bash
+mc mirror --overwrite obj/drops/ backup-target/drops-mirror/
+```
+
+(Exclude `_pgbackups/` from the app-data mirror if you split them.)
+
+## Disaster scenarios
+
+| Scenario | Recovery |
+|---|---|
+| Postgres pod crash / reschedule | StatefulSet reattaches the PVC; ~1–2 min downtime; no data loss. |
+| PVC lost / corrupted | Recreate StatefulSet, restore latest nightly dump (above). Data since last dump is lost. |
+| Accidental `drops` table data loss | Restore latest dump; or `pg_restore` a single table from a dump. |
+| Namespace deleted | PVC has `Prune=false,Delete=false`; recreate Applications, PVC re-binds, app recovers. Backups in object storage are independent. |
+| Object storage bucket lost | UpCloud-managed (replicated). If the IAM key is rotated, update `forte-drop-secrets` (re-seal). |

infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml

@@ -0,0 +1,40 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forte-drop-postgresql
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+  labels:
+    app.kubernetes.io/name: forte-drop-postgresql
+    app.kubernetes.io/part-of: apps
+    app.kubernetes.io/managed-by: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+
+  source:
+    repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
+    targetRevision: HEAD
+    path: infra/overlays/upc-dev/forte-drop-postgresql/resources
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: forte-drop
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    syncOptions:
+    - CreateNamespace=true
+    - Validate=true
+    - ServerSideApply=true
+
+  ignoreDifferences:
+  - group: apps
+    kind: StatefulSet
+    jsonPointers:
+    - /spec/volumeClaimTemplates

infra/overlays/upc-dev/forte-drop-postgresql/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- forte-drop-postgresql.yaml

infra/overlays/upc-dev/forte-drop-postgresql/resources/forte-drop-pg-creds-sealed.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: forte-drop-pg-creds
+  namespace: forte-drop
+spec:
+  encryptedData:
+    pgpassword: AgBYokuQRCTmPGC8soB7n8W39nmSAZDTV97i77NmdMh5ndaZNtCtXxhMcpM2z8kaGv2tCWIh47dr38a5tGPZ0TsmeEQOF9Rbbtq1fyR7pJwT8S+N2Z2zu354wNWQlltEAVvTvthe2wTer/BpyRofeSZprxihmfNpuHUP8rLsnIXln5tOWDzJ8hnRWoYZFITaihC2qrJj/kFE0Rfdcmzt1tSq3jCB/rWijVaJF9XSh4rzQoqZDiUNjDPUjyERILw59JWU4zf9OKcqNHDnmpXBR4LSjLhd9waN6ElEzO4gGcVaHISKrTwewX1ONwPHDnw6lqkQObyBPx8aUsGzxLkUhNDtvIYDkB4BKWP5Qu4bNcSztIbrxi6l7Lr/DWC9qTbKm1/p83rc6r8VqRMURUyQg8/vBlCHOIUbZ8DM1OfNlMd8gvcSkaxEVIdCDUjguCvE3cGyG4cqv2unllZQ+9417WwLNJecT6x1EL3nQyAlK5c9vUIbcVyaFlbSUcGB7xmPgZrZ6/3RDyOH6Tmew1ssV9gLvdaehscUE0fjnnFnJpczkwdyxIOSNLIkjlWetCKEbhowJbzk05h3M2p6XQQOuNTnsYjAADMGD72GUgAQlY8KXmDELtv09KELcXbeYS4gABPpMrVmvZymq8lqQ13Py8o+cIqbrU5V86WxASTfQ5gMo/ymYabuhTBIapcnaKR1dFCCfu8deh5f2HJ6/1NjdWR+XvEshg+EF5OkTUInukX4vA==
+    pgusername: AgCs6vyQ8CIv5OneP/jMltIPGdZQbpq/BFmQM1mkBD61Ve+anzve5K0Gkg+zsNfbZf0pOPAXtu4C4aL1Lwv7gqpoe4Hp/UEb/X9uLfJ1b8ZitmM1XsPmmSiCskHjrc2BLkAvfrVIXkHc3LOY2uZ/E5stc6Ss2WFE8/uzzVXW0B8fdEK0criludQ8iwR1gypulEcDNomXgkK/1gmmCWosUcVv4jDMDhqBD+b9WYnBB6J73gUclWVMvYDFdNas2PuoRzu5Twc9TAZrTxN5lvLOXAonOo0YiUbUhEC83sfMWYDT5/9OxqcJhAxtgFe9j83MpCwLSwfeLZm7UsUapWDb60MxPJLGvoGD/ZOhkeYt/YCZYROa57TMslVIL5YU1KCiNWvtRjIqnvdiBxI7MRvPUfAoawS4ktT5PDhTTfrixFbaF95jul2kKBXV+OYB1UNsFhcCgZx9rzYRt4lNmBv4m4HeXIp3EYY8VlGLQ45BVVqjJ4QkISvb7ifQWH1aPMQllj+J3GwW0KJN0dEgsh1LT+C7W5I5mq461NOTF1eih/XRBeuPoLlgApxiGXvFCTx8lji2/JIdOaqcg29hdabSprxa0YMStChi2pbtHhRzAuFCp8mInGt8Q406vu67Y4/51yuwI40YeDVu0lf010TB+/v2Zy3OrNyjlqrD5JNynsLuRl3UhuAKC14Xhg/MiDLvTzfsYE8aog==
+  template:
+    metadata:
+      name: forte-drop-pg-creds
+      namespace: forte-drop

infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- postgresql.yaml
+- forte-drop-pg-creds-sealed.yaml
+- pg-backup-cronjob.yaml

infra/overlays/upc-dev/forte-drop-postgresql/resources/pg-backup-cronjob.yaml

@@ -0,0 +1,93 @@
+# Nightly logical backup of the forte-drop Postgres → UpCloud Managed Object Storage.
+# Dumps to s3://drops/_pgbackups/ (the `_` prefix is collision-proof: app slugs match
+# /^[a-z0-9][a-z0-9-]{0,62}$/ and can never start with `_`). Retains 30 days.
+#
+# Pod shape: initContainer pg_dump → shared emptyDir → mc upload + retention prune.
+# Both images pinned. S3 creds reuse forte-drop-secrets (the app's UpCloud user has
+# s3:* on the drops bucket). PG creds from forte-drop-pg-creds.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: forte-drop-pg-backup
+  namespace: forte-drop
+  labels:
+    app.kubernetes.io/name: postgresql
+    app.kubernetes.io/instance: forte-drop
+    app.kubernetes.io/component: backup
+spec:
+  schedule: "0 2 * * *"           # 02:00 UTC daily
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: postgresql
+            app.kubernetes.io/instance: forte-drop
+            app.kubernetes.io/component: backup
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            fsGroup: 65532
+          volumes:
+          - name: work
+            emptyDir: {}
+          initContainers:
+          - name: dump
+            image: postgres:16-alpine
+            command:
+            - sh
+            - -c
+            - |
+              set -euo pipefail
+              TS=$(date -u +%Y%m%dT%H%M%SZ)
+              echo "dumping to /work/forte-drop-${TS}.sql.gz"
+              PGPASSWORD="$PGPASSWORD" pg_dump \
+                -h forte-drop-postgresql.forte-drop.svc \
+                -p 5432 -U "$PGUSER" -d drops \
+                --no-owner --no-privileges \
+                | gzip -9 > "/work/forte-drop-${TS}.sql.gz"
+              echo "dump complete: $(ls -lh /work/)"
+            env:
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef: { name: forte-drop-pg-creds, key: pgusername }
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef: { name: forte-drop-pg-creds, key: pgpassword }
+            volumeMounts:
+            - name: work
+              mountPath: /work
+          containers:
+          - name: upload
+            image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z
+            command:
+            - sh
+            - -c
+            - |
+              set -euo pipefail
+              mc alias set obj "$S3_ENDPOINT" "$S3_KEY" "$S3_SECRET"
+              mc cp /work/*.sql.gz "obj/${S3_BUCKET}/_pgbackups/"
+              echo "uploaded. pruning backups older than 30d:"
+              mc rm --recursive --force --older-than 30d "obj/${S3_BUCKET}/_pgbackups/" || true
+              echo "backup retention pass complete"
+            env:
+            - name: S3_ENDPOINT
+              valueFrom:
+                secretKeyRef: { name: forte-drop-secrets, key: S3_ENDPOINT }
+            - name: S3_BUCKET
+              value: "drops"
+            - name: S3_KEY
+              valueFrom:
+                secretKeyRef: { name: forte-drop-secrets, key: S3_KEY }
+            - name: S3_SECRET
+              valueFrom:
+                secretKeyRef: { name: forte-drop-secrets, key: S3_SECRET }
+            volumeMounts:
+            - name: work
+              mountPath: /work

infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml

@@ -0,0 +1,105 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: forte-drop-postgresql
+  namespace: forte-drop
+  labels:
+    app.kubernetes.io/name: postgresql
+    app.kubernetes.io/instance: forte-drop
+    app.kubernetes.io/component: database
+spec:
+  type: ClusterIP
+  ports:
+  - name: tcp-postgresql
+    port: 5432
+    targetPort: tcp-postgresql
+  selector:
+    app.kubernetes.io/name: postgresql
+    app.kubernetes.io/instance: forte-drop
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: forte-drop-postgresql
+  namespace: forte-drop
+  labels:
+    app.kubernetes.io/name: postgresql
+    app.kubernetes.io/instance: forte-drop
+    app.kubernetes.io/component: database
+spec:
+  serviceName: forte-drop-postgresql
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: postgresql
+      app.kubernetes.io/instance: forte-drop
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: postgresql
+        app.kubernetes.io/instance: forte-drop
+        app.kubernetes.io/component: database
+    spec:
+      containers:
+      - name: postgresql
+        image: postgres:16-alpine
+        # NOTE: no securityContext. The official postgres image's entrypoint must
+        # start as root to chown a fresh /var/lib/postgresql/data, then drops to
+        # the postgres user (uid 70 in alpine) via gosu. Forcing runAsNonRoot here
+        # breaks initdb on a fresh PVC. Matches the vaultwarden-postgresql pattern.
+        ports:
+        - name: tcp-postgresql
+          containerPort: 5432
+        env:
+        - name: POSTGRES_USER
+          valueFrom:
+            secretKeyRef:
+              name: forte-drop-pg-creds
+              key: pgusername
+        - name: POSTGRES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: forte-drop-pg-creds
+              key: pgpassword
+        - name: POSTGRES_DB
+          value: drops
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/postgresql/data
+        livenessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d drops
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          exec:
+            command:
+            - sh
+            - -c
+            - pg_isready -U "$POSTGRES_USER" -d drops
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+      annotations:
+        argocd.argoproj.io/sync-options: Prune=false,Delete=false
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: upcloud-block-storage-maxiops
+      resources:
+        requests:
+          storage: 5Gi

infra/overlays/upc-dev/kustomization.yaml

@@ -4,6 +4,7 @@ resources:
 - ../../base
 - vaultwarden-postgresql
 - vaultwarden
+- forte-drop-postgresql
 
 # No patches needed — base already has "upc-dev" paths
 # upc-dev is the default/base cluster