From 3ce93017f9bff49a9bd073f75dab799d002dcddb Mon Sep 17 00:00:00 2001 From: Sten Date: Thu, 28 May 2026 14:33:19 +0200 Subject: [PATCH 1/7] feat(infra): forte-drop postgres + minio for upc-dev Two new ArgoCD Applications: - forte-drop-postgresql: in-cluster Postgres 16 StatefulSet, 5Gi PVC, POSTGRES_DB=drops, creds from forte-drop-pg-creds SealedSecret. - forte-drop-minio: in-cluster MinIO StatefulSet, 20Gi PVC, bootstrap Job creates the 'drops' bucket post-sync, creds from forte-drop-minio-creds SealedSecret. Both live in namespace 'forte-drop'. Mirrors the Vaultwarden pattern. Sealed secrets are added in a follow-up commit by the maintainer: kubeseal --fetch-cert > pub.pem kubeseal --cert pub.pem --format yaml < private/forte-drop-pg-creds.yaml > \ infra/overlays/upc-dev/forte-drop-postgresql/resources/forte-drop-pg-creds-sealed.yaml kubeseal --cert pub.pem --format yaml < private/forte-drop-minio-creds.yaml > \ infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml --- .../forte-drop-minio/forte-drop-minio.yaml | 40 +++++ .../forte-drop-minio/kustomization.yaml | 4 + .../resources/kustomization.yaml | 5 + .../forte-drop-minio/resources/minio.yaml | 146 ++++++++++++++++++ .../forte-drop-postgresql.yaml | 45 ++++++ .../forte-drop-postgresql/kustomization.yaml | 4 + .../resources/kustomization.yaml | 5 + .../resources/postgresql.yaml | 99 ++++++++++++ infra/overlays/upc-dev/kustomization.yaml | 2 + 9 files changed, 350 insertions(+) create mode 100644 infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/kustomization.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml b/infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml new file mode 100644 index 0000000..f8afb0e --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml @@ -0,0 +1,40 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: forte-drop-minio + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + labels: + app.kubernetes.io/name: forte-drop-minio + app.kubernetes.io/part-of: apps + app.kubernetes.io/managed-by: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + + source: + repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git + targetRevision: HEAD + path: infra/overlays/upc-dev/forte-drop-minio/resources + + destination: + server: https://kubernetes.default.svc + namespace: forte-drop + + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=true + - Validate=true + - ServerSideApply=true + + ignoreDifferences: + - group: apps + kind: StatefulSet + jsonPointers: + - /spec/volumeClaimTemplates diff --git a/infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml new file mode 100644 index 0000000..edb8209 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- forte-drop-minio.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml new file mode 100644 index 0000000..30fb6c0 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- minio.yaml +- forte-drop-minio-creds-sealed.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml new file mode 100644 index 0000000..51bde97 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml @@ -0,0 +1,146 @@ +apiVersion: v1 +kind: Service +metadata: + name: forte-drop-minio + namespace: forte-drop + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: object-storage +spec: + type: ClusterIP + ports: + - name: http-api + port: 9000 + targetPort: http-api + - name: http-console + port: 9001 + targetPort: http-console + selector: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: forte-drop +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: forte-drop-minio + namespace: forte-drop + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: object-storage +spec: + serviceName: forte-drop-minio + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: forte-drop + template: + metadata: + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: object-storage + spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + args: + - server + - /data + - --console-address + - :9001 + ports: + - name: http-api + containerPort: 9000 + - name: http-console + containerPort: 9001 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: forte-drop-minio-creds + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: forte-drop-minio-creds + key: root-password + volumeMounts: + - name: data + mountPath: /data + livenessProbe: + httpGet: + path: /minio/health/live + port: http-api + initialDelaySeconds: 30 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /minio/health/ready + port: http-api + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: upcloud-block-storage-maxiops + resources: + requests: + storage: 20Gi +--- +# Bootstrap job — creates the 'drops' bucket once MinIO is reachable. +# Idempotent: `mc mb --ignore-existing` skips if bucket already exists. +apiVersion: batch/v1 +kind: Job +metadata: + name: forte-drop-minio-bootstrap + namespace: forte-drop + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: bootstrap + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded +spec: + backoffLimit: 5 + template: + spec: + restartPolicy: OnFailure + containers: + - name: mc + image: quay.io/minio/mc:latest + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: forte-drop-minio-creds + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: forte-drop-minio-creds + key: root-password + command: + - sh + - -c + - | + set -e + until mc alias set local http://forte-drop-minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do + echo "waiting for minio..." + sleep 2 + done + mc mb --ignore-existing local/drops + echo "bucket 'drops' ready" diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml new file mode 100644 index 0000000..31be7e1 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: forte-drop +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: forte-drop-postgresql + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" + labels: + app.kubernetes.io/name: forte-drop-postgresql + app.kubernetes.io/part-of: apps + app.kubernetes.io/managed-by: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + + source: + repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git + targetRevision: HEAD + path: infra/overlays/upc-dev/forte-drop-postgresql/resources + + destination: + server: https://kubernetes.default.svc + namespace: forte-drop + + syncPolicy: + automated: + prune: true + selfHeal: true + allowEmpty: false + syncOptions: + - CreateNamespace=true + - Validate=true + - ServerSideApply=true + + ignoreDifferences: + - group: apps + kind: StatefulSet + jsonPointers: + - /spec/volumeClaimTemplates diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/kustomization.yaml new file mode 100644 index 0000000..28749fb --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- forte-drop-postgresql.yaml diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml new file mode 100644 index 0000000..681bbcf --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- postgresql.yaml +- forte-drop-pg-creds-sealed.yaml diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml new file mode 100644 index 0000000..c1b26d1 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml @@ -0,0 +1,99 @@ +apiVersion: v1 +kind: Service +metadata: + name: forte-drop-postgresql + namespace: forte-drop + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: database +spec: + type: ClusterIP + ports: + - name: tcp-postgresql + port: 5432 + targetPort: tcp-postgresql + selector: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: forte-drop-postgresql + namespace: forte-drop + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: database +spec: + serviceName: forte-drop-postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: database + spec: + containers: + - name: postgresql + image: postgres:16-alpine + ports: + - name: tcp-postgresql + containerPort: 5432 + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: forte-drop-pg-creds + key: pgusername + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: forte-drop-pg-creds + key: pgpassword + - name: POSTGRES_DB + value: drops + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + livenessProbe: + exec: + command: + - sh + - -c + - pg_isready -U "$POSTGRES_USER" -d drops + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: + - sh + - -c + - pg_isready -U "$POSTGRES_USER" -d drops + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + storageClassName: upcloud-block-storage-maxiops + resources: + requests: + storage: 5Gi diff --git a/infra/overlays/upc-dev/kustomization.yaml b/infra/overlays/upc-dev/kustomization.yaml index fac7510..f36009e 100644 --- a/infra/overlays/upc-dev/kustomization.yaml +++ b/infra/overlays/upc-dev/kustomization.yaml @@ -4,6 +4,8 @@ resources: - ../../base - vaultwarden-postgresql - vaultwarden +- forte-drop-postgresql +- forte-drop-minio # No patches needed — base already has "upc-dev" paths # upc-dev is the default/base cluster -- 2.49.1 From 416615a9e05846b54b180ee7d4b6de8a44d58490 Mon Sep 17 00:00:00 2001 From: Sten Date: Thu, 28 May 2026 15:56:24 +0200 Subject: [PATCH 2/7] feat(infra): add forte-drop sealed secrets Pg and minio credentials sealed against upc-dev sealed-secrets-controller. --- .../resources/forte-drop-minio-creds-sealed.yaml | 14 ++++++++++++++ .../resources/forte-drop-pg-creds-sealed.yaml | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/resources/forte-drop-pg-creds-sealed.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml new file mode 100644 index 0000000..dc0d454 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: forte-drop-minio-creds + namespace: forte-drop +spec: + encryptedData: + root-password: AgCaEHfMk/fClLDpvNvAQKIFPHRkZZVqw7/Ct4GO7v0kVpdMbQiisJ+dkYlpFsOCflQ7HaruF8Cf8OiGX5qlfUx5R4WG+msVvtgGd9zK3DJ2w0DiD5aIlyZQkAspbsUPJSUmfPGb1wrKIlCLd3vOPZJcGr1/nH9Dr2VWJJuEDvootJHvXzq3PZlJ9aGDglvyOWsibHUKQdFIhdg1Ajs3kGYIZYmVsAIyqjSXeo1jsB9nUiFy2ufKO5STiF6KQ+EPOjv+tyfhBB6Tdu46d82BaFY3CML/HV2lbkPlsCkByIb16g+Sbv4yTaSB/VVSLeqD2MyD9J4QRB5wd2luEFjpopW0tXUCqks/CoFX25gTd6TF0joaYXFuMI4Pf9EgBpXY0nZGW+K+Kqo8sHaSWZR6CT1wubHzky1GN8+8GTj1nGvqRSgVOIyCJqP+CylAuhEzYgGs9OCwX9iulBlQjQnpP6dsPhp/QB4C9VH4xFU2kc1cbKWiGFMpyXxMxBFGuEKjz+KK577HlwxqITcaMhBxvIS/PimOrAL40vl65dDtjUaHFez4Hv6u13YO+F21m5J9ChFr+l4X8i96GPoZQ4fshOCmjkZ/eNHZTMLA0iViD7Zqqj1l4rgd3PW+A9Sd1kq1aeRPoyF3FHBi17DulR+2qgrdO0QrLnKVRMRDmxaNvleyLJVX3NwJn2yjZrlfQPqzCxTmAzBTnUMCwK5WOtzVQpH9TP8xC4yj1sxhFBv1cmH5Jg== + root-user: AgDVJTL+2vSqkv6jMERD4TPUMA9/41WA0+9tpW0Had3RpkvDUAe2dCTcIHeLFra0XMxZE7z9TbJqP1d0rDsk5SSoSqRq+wR1gizjMuiBRSt8icVgClHvNxwXfoGz4Z0FH6IR9M5G+o36hy5kkprJcv1OJ4u+ddXs2sv3zEGUaS6Am8jscij2IJYSDK3jExpv/UEfKKqWcHrfbcoIILdk3t+FGjX9SiTC6s7DEBgbyaVjQavvA+q4GasLm2vTaFPE+G6rb10LB3Yq4pF8h7AplBZwIylEeuaVE1pivLkNHMqTSZbNaV3DZRUGTa4EwQoM8dC1CKuhBONNt25SfQ3n/Aiv6gl1eKCSEcoq4zUDHGsa4iTyqT5VyoKI6VZ0reEJVaut/4wavuc/dNkBHiJ9Y1IZwjpYLsTy+zrnX2vv1qYLBKg59AlEg73lB0HcLFkQBVvR+dkE1pKHFDrR5BDww6RykCF4/1eGS23MuPr0SudEejYmm/P/5FrBMQ1mB8ZhPw074brxkLtgW4qgsLlSN0GkFcqbR7b4pI4SPMmXBO+p9iX6dI8BgA8ZoL84jrmHlUMXeCnHiN5RV8nNt1b0DO7LK0cyHe1YFH4vHpIDXv9K1wxLuwLfjk4xCP/e6wlximdPzaKx6uBCW8xRs0BCnoLsWP7uXnF+R/ELQlyHwsrVyMXnODewhsrcqnqORm8kOjxdR8w2hkqCFqYj3c6h5nkA + template: + metadata: + name: forte-drop-minio-creds + namespace: forte-drop diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/forte-drop-pg-creds-sealed.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/forte-drop-pg-creds-sealed.yaml new file mode 100644 index 0000000..ad22994 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/forte-drop-pg-creds-sealed.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: forte-drop-pg-creds + namespace: forte-drop +spec: + encryptedData: + pgpassword: AgBYokuQRCTmPGC8soB7n8W39nmSAZDTV97i77NmdMh5ndaZNtCtXxhMcpM2z8kaGv2tCWIh47dr38a5tGPZ0TsmeEQOF9Rbbtq1fyR7pJwT8S+N2Z2zu354wNWQlltEAVvTvthe2wTer/BpyRofeSZprxihmfNpuHUP8rLsnIXln5tOWDzJ8hnRWoYZFITaihC2qrJj/kFE0Rfdcmzt1tSq3jCB/rWijVaJF9XSh4rzQoqZDiUNjDPUjyERILw59JWU4zf9OKcqNHDnmpXBR4LSjLhd9waN6ElEzO4gGcVaHISKrTwewX1ONwPHDnw6lqkQObyBPx8aUsGzxLkUhNDtvIYDkB4BKWP5Qu4bNcSztIbrxi6l7Lr/DWC9qTbKm1/p83rc6r8VqRMURUyQg8/vBlCHOIUbZ8DM1OfNlMd8gvcSkaxEVIdCDUjguCvE3cGyG4cqv2unllZQ+9417WwLNJecT6x1EL3nQyAlK5c9vUIbcVyaFlbSUcGB7xmPgZrZ6/3RDyOH6Tmew1ssV9gLvdaehscUE0fjnnFnJpczkwdyxIOSNLIkjlWetCKEbhowJbzk05h3M2p6XQQOuNTnsYjAADMGD72GUgAQlY8KXmDELtv09KELcXbeYS4gABPpMrVmvZymq8lqQ13Py8o+cIqbrU5V86WxASTfQ5gMo/ymYabuhTBIapcnaKR1dFCCfu8deh5f2HJ6/1NjdWR+XvEshg+EF5OkTUInukX4vA== + pgusername: AgCs6vyQ8CIv5OneP/jMltIPGdZQbpq/BFmQM1mkBD61Ve+anzve5K0Gkg+zsNfbZf0pOPAXtu4C4aL1Lwv7gqpoe4Hp/UEb/X9uLfJ1b8ZitmM1XsPmmSiCskHjrc2BLkAvfrVIXkHc3LOY2uZ/E5stc6Ss2WFE8/uzzVXW0B8fdEK0criludQ8iwR1gypulEcDNomXgkK/1gmmCWosUcVv4jDMDhqBD+b9WYnBB6J73gUclWVMvYDFdNas2PuoRzu5Twc9TAZrTxN5lvLOXAonOo0YiUbUhEC83sfMWYDT5/9OxqcJhAxtgFe9j83MpCwLSwfeLZm7UsUapWDb60MxPJLGvoGD/ZOhkeYt/YCZYROa57TMslVIL5YU1KCiNWvtRjIqnvdiBxI7MRvPUfAoawS4ktT5PDhTTfrixFbaF95jul2kKBXV+OYB1UNsFhcCgZx9rzYRt4lNmBv4m4HeXIp3EYY8VlGLQ45BVVqjJ4QkISvb7ifQWH1aPMQllj+J3GwW0KJN0dEgsh1LT+C7W5I5mq461NOTF1eih/XRBeuPoLlgApxiGXvFCTx8lji2/JIdOaqcg29hdabSprxa0YMStChi2pbtHhRzAuFCp8mInGt8Q406vu67Y4/51yuwI40YeDVu0lf010TB+/v2Zy3OrNyjlqrD5JNynsLuRl3UhuAKC14Xhg/MiDLvTzfsYE8aog== + template: + metadata: + name: forte-drop-pg-creds + namespace: forte-drop -- 2.49.1 From 69848e42f02ecedfe6af1201e0b482fc837205df Mon Sep 17 00:00:00 2001 From: Sten Date: Thu, 28 May 2026 16:05:48 +0200 Subject: [PATCH 3/7] fix(infra): pin minio/mc tags + add postgres securityContext + harden bootstrap script Address ai-review feedback on PR #17: - Pin quay.io/minio/minio and mc to specific RELEASE tags (Renovate will bump). 'latest' is unpredictable in GitOps. - Bootstrap script: set -e -> set -euo pipefail. - Postgres container: runAsNonRoot, uid/gid 999, drop ALL caps, no privilege escalation. Matches PSS restricted profile. --- .../overlays/upc-dev/forte-drop-minio/resources/minio.yaml | 6 +++--- .../forte-drop-postgresql/resources/postgresql.yaml | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml index 51bde97..69d1e3a 100644 --- a/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml +++ b/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml @@ -45,7 +45,7 @@ spec: spec: containers: - name: minio - image: quay.io/minio/minio:latest + image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z args: - server - /data @@ -121,7 +121,7 @@ spec: restartPolicy: OnFailure containers: - name: mc - image: quay.io/minio/mc:latest + image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z env: - name: MINIO_ROOT_USER valueFrom: @@ -137,7 +137,7 @@ spec: - sh - -c - | - set -e + set -euo pipefail until mc alias set local http://forte-drop-minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do echo "waiting for minio..." sleep 2 diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml index c1b26d1..dac60e0 100644 --- a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml @@ -43,6 +43,13 @@ spec: containers: - name: postgresql image: postgres:16-alpine + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] ports: - name: tcp-postgresql containerPort: 5432 -- 2.49.1 From 47d1f1ec39cf318477e3cad0fa5aaaba599d66ae Mon Sep 17 00:00:00 2001 From: Sten Date: Thu, 28 May 2026 16:13:08 +0200 Subject: [PATCH 4/7] fix(infra): drop bad postgres securityContext + un-own shared namespace Address Codex review on PR #17: [P1] Postgres official image's entrypoint requires root to chown a fresh PVC, then drops to the postgres user via gosu. Forcing runAsNonRoot+runAsUser=999 blocks the chown and initdb fails on a fresh volume. Drop the securityContext; matches the existing vaultwarden-postgresql pattern. [P2] The forte-drop namespace was declared as a managed resource in the postgres Application. Since minio lives in the same namespace from a separate Application, an Argo prune of the pg app would delete the namespace and cascade-delete minio. Remove the Namespace resource; rely on syncOptions: CreateNamespace=true on both apps (already set). --- .../forte-drop-postgresql/resources/postgresql.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml index dac60e0..659ba9b 100644 --- a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml @@ -43,13 +43,10 @@ spec: containers: - name: postgresql image: postgres:16-alpine - securityContext: - runAsNonRoot: true - runAsUser: 999 - runAsGroup: 999 - allowPrivilegeEscalation: false - capabilities: - drop: [ALL] + # NOTE: no securityContext. The official postgres image's entrypoint must + # start as root to chown a fresh /var/lib/postgresql/data, then drops to + # the postgres user (uid 70 in alpine) via gosu. Forcing runAsNonRoot here + # breaks initdb on a fresh PVC. Matches the vaultwarden-postgresql pattern. ports: - name: tcp-postgresql containerPort: 5432 -- 2.49.1 From 178bf8cc78b4cb47b6930c16cf9fc288d6e57816 Mon Sep 17 00:00:00 2001 From: Sten Date: Thu, 28 May 2026 16:13:31 +0200 Subject: [PATCH 5/7] fix(infra): un-own forte-drop namespace from postgres app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to 47d1f1e — the previous commit only updated postgres' securityContext; this drops the explicit Namespace resource as the Codex review flagged. Both apps still get the namespace created via syncOptions: CreateNamespace=true. --- .../upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml index 31be7e1..7a226cf 100644 --- a/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml +++ b/infra/overlays/upc-dev/forte-drop-postgresql/forte-drop-postgresql.yaml @@ -1,8 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: forte-drop ---- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: -- 2.49.1 From dd9819bdbe387ef11eefecad8988f801531150ee Mon Sep 17 00:00:00 2001 From: Sten Date: Fri, 29 May 2026 09:28:51 +0200 Subject: [PATCH 6/7] feat(infra): drop in-cluster minio, add pg backup + PVC protection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PROD: object storage moves to UpCloud Managed Object Storage (existing drops bucket) instead of single-node in-cluster MinIO — durable, UpCloud-replicated, no PVC to back up. - Remove forte-drop-minio StatefulSet entirely. - Add forte-drop-pg-backup CronJob: nightly pg_dump -> gzip -> upload to s3://drops/_pgbackups/ (collision-proof prefix), 30-day retention. Reuses forte-drop-secrets S3 creds (app user has s3:* on drops). - PVC prune/delete protection on the postgres volumeClaimTemplate. --- .../forte-drop-minio/forte-drop-minio.yaml | 40 ----- .../forte-drop-minio/kustomization.yaml | 4 - .../forte-drop-minio-creds-sealed.yaml | 14 -- .../resources/kustomization.yaml | 5 - .../forte-drop-minio/resources/minio.yaml | 146 ------------------ .../resources/kustomization.yaml | 1 + .../resources/pg-backup-cronjob.yaml | 93 +++++++++++ .../resources/postgresql.yaml | 2 + infra/overlays/upc-dev/kustomization.yaml | 1 - 9 files changed, 96 insertions(+), 210 deletions(-) delete mode 100644 infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml delete mode 100644 infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml delete mode 100644 infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml delete mode 100644 infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml delete mode 100644 infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/resources/pg-backup-cronjob.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml b/infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml deleted file mode 100644 index f8afb0e..0000000 --- a/infra/overlays/upc-dev/forte-drop-minio/forte-drop-minio.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: forte-drop-minio - namespace: argocd - annotations: - argocd.argoproj.io/sync-wave: "0" - labels: - app.kubernetes.io/name: forte-drop-minio - app.kubernetes.io/part-of: apps - app.kubernetes.io/managed-by: argocd - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - project: default - - source: - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git - targetRevision: HEAD - path: infra/overlays/upc-dev/forte-drop-minio/resources - - destination: - server: https://kubernetes.default.svc - namespace: forte-drop - - syncPolicy: - automated: - prune: true - selfHeal: true - allowEmpty: false - syncOptions: - - CreateNamespace=true - - Validate=true - - ServerSideApply=true - - ignoreDifferences: - - group: apps - kind: StatefulSet - jsonPointers: - - /spec/volumeClaimTemplates diff --git a/infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml deleted file mode 100644 index edb8209..0000000 --- a/infra/overlays/upc-dev/forte-drop-minio/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- forte-drop-minio.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml deleted file mode 100644 index dc0d454..0000000 --- a/infra/overlays/upc-dev/forte-drop-minio/resources/forte-drop-minio-creds-sealed.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - name: forte-drop-minio-creds - namespace: forte-drop -spec: - encryptedData: - root-password: AgCaEHfMk/fClLDpvNvAQKIFPHRkZZVqw7/Ct4GO7v0kVpdMbQiisJ+dkYlpFsOCflQ7HaruF8Cf8OiGX5qlfUx5R4WG+msVvtgGd9zK3DJ2w0DiD5aIlyZQkAspbsUPJSUmfPGb1wrKIlCLd3vOPZJcGr1/nH9Dr2VWJJuEDvootJHvXzq3PZlJ9aGDglvyOWsibHUKQdFIhdg1Ajs3kGYIZYmVsAIyqjSXeo1jsB9nUiFy2ufKO5STiF6KQ+EPOjv+tyfhBB6Tdu46d82BaFY3CML/HV2lbkPlsCkByIb16g+Sbv4yTaSB/VVSLeqD2MyD9J4QRB5wd2luEFjpopW0tXUCqks/CoFX25gTd6TF0joaYXFuMI4Pf9EgBpXY0nZGW+K+Kqo8sHaSWZR6CT1wubHzky1GN8+8GTj1nGvqRSgVOIyCJqP+CylAuhEzYgGs9OCwX9iulBlQjQnpP6dsPhp/QB4C9VH4xFU2kc1cbKWiGFMpyXxMxBFGuEKjz+KK577HlwxqITcaMhBxvIS/PimOrAL40vl65dDtjUaHFez4Hv6u13YO+F21m5J9ChFr+l4X8i96GPoZQ4fshOCmjkZ/eNHZTMLA0iViD7Zqqj1l4rgd3PW+A9Sd1kq1aeRPoyF3FHBi17DulR+2qgrdO0QrLnKVRMRDmxaNvleyLJVX3NwJn2yjZrlfQPqzCxTmAzBTnUMCwK5WOtzVQpH9TP8xC4yj1sxhFBv1cmH5Jg== - root-user: AgDVJTL+2vSqkv6jMERD4TPUMA9/41WA0+9tpW0Had3RpkvDUAe2dCTcIHeLFra0XMxZE7z9TbJqP1d0rDsk5SSoSqRq+wR1gizjMuiBRSt8icVgClHvNxwXfoGz4Z0FH6IR9M5G+o36hy5kkprJcv1OJ4u+ddXs2sv3zEGUaS6Am8jscij2IJYSDK3jExpv/UEfKKqWcHrfbcoIILdk3t+FGjX9SiTC6s7DEBgbyaVjQavvA+q4GasLm2vTaFPE+G6rb10LB3Yq4pF8h7AplBZwIylEeuaVE1pivLkNHMqTSZbNaV3DZRUGTa4EwQoM8dC1CKuhBONNt25SfQ3n/Aiv6gl1eKCSEcoq4zUDHGsa4iTyqT5VyoKI6VZ0reEJVaut/4wavuc/dNkBHiJ9Y1IZwjpYLsTy+zrnX2vv1qYLBKg59AlEg73lB0HcLFkQBVvR+dkE1pKHFDrR5BDww6RykCF4/1eGS23MuPr0SudEejYmm/P/5FrBMQ1mB8ZhPw074brxkLtgW4qgsLlSN0GkFcqbR7b4pI4SPMmXBO+p9iX6dI8BgA8ZoL84jrmHlUMXeCnHiN5RV8nNt1b0DO7LK0cyHe1YFH4vHpIDXv9K1wxLuwLfjk4xCP/e6wlximdPzaKx6uBCW8xRs0BCnoLsWP7uXnF+R/ELQlyHwsrVyMXnODewhsrcqnqORm8kOjxdR8w2hkqCFqYj3c6h5nkA - template: - metadata: - name: forte-drop-minio-creds - namespace: forte-drop diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml deleted file mode 100644 index 30fb6c0..0000000 --- a/infra/overlays/upc-dev/forte-drop-minio/resources/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- minio.yaml -- forte-drop-minio-creds-sealed.yaml diff --git a/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml b/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml deleted file mode 100644 index 69d1e3a..0000000 --- a/infra/overlays/upc-dev/forte-drop-minio/resources/minio.yaml +++ /dev/null @@ -1,146 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: forte-drop-minio - namespace: forte-drop - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/instance: forte-drop - app.kubernetes.io/component: object-storage -spec: - type: ClusterIP - ports: - - name: http-api - port: 9000 - targetPort: http-api - - name: http-console - port: 9001 - targetPort: http-console - selector: - app.kubernetes.io/name: minio - app.kubernetes.io/instance: forte-drop ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: forte-drop-minio - namespace: forte-drop - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/instance: forte-drop - app.kubernetes.io/component: object-storage -spec: - serviceName: forte-drop-minio - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: minio - app.kubernetes.io/instance: forte-drop - template: - metadata: - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/instance: forte-drop - app.kubernetes.io/component: object-storage - spec: - containers: - - name: minio - image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z - args: - - server - - /data - - --console-address - - :9001 - ports: - - name: http-api - containerPort: 9000 - - name: http-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: forte-drop-minio-creds - key: root-user - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: forte-drop-minio-creds - key: root-password - volumeMounts: - - name: data - mountPath: /data - livenessProbe: - httpGet: - path: /minio/health/live - port: http-api - initialDelaySeconds: 30 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /minio/health/ready - port: http-api - initialDelaySeconds: 5 - periodSeconds: 5 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 1Gi - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - storageClassName: upcloud-block-storage-maxiops - resources: - requests: - storage: 20Gi ---- -# Bootstrap job — creates the 'drops' bucket once MinIO is reachable. -# Idempotent: `mc mb --ignore-existing` skips if bucket already exists. -apiVersion: batch/v1 -kind: Job -metadata: - name: forte-drop-minio-bootstrap - namespace: forte-drop - labels: - app.kubernetes.io/name: minio - app.kubernetes.io/instance: forte-drop - app.kubernetes.io/component: bootstrap - annotations: - argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: HookSucceeded -spec: - backoffLimit: 5 - template: - spec: - restartPolicy: OnFailure - containers: - - name: mc - image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: forte-drop-minio-creds - key: root-user - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: forte-drop-minio-creds - key: root-password - command: - - sh - - -c - - | - set -euo pipefail - until mc alias set local http://forte-drop-minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do - echo "waiting for minio..." - sleep 2 - done - mc mb --ignore-existing local/drops - echo "bucket 'drops' ready" diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml index 681bbcf..7b5a754 100644 --- a/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/kustomization.yaml @@ -3,3 +3,4 @@ kind: Kustomization resources: - postgresql.yaml - forte-drop-pg-creds-sealed.yaml +- pg-backup-cronjob.yaml diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/pg-backup-cronjob.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/pg-backup-cronjob.yaml new file mode 100644 index 0000000..4304424 --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/pg-backup-cronjob.yaml @@ -0,0 +1,93 @@ +# Nightly logical backup of the forte-drop Postgres → UpCloud Managed Object Storage. +# Dumps to s3://drops/_pgbackups/ (the `_` prefix is collision-proof: app slugs match +# /^[a-z0-9][a-z0-9-]{0,62}$/ and can never start with `_`). Retains 30 days. +# +# Pod shape: initContainer pg_dump → shared emptyDir → mc upload + retention prune. +# Both images pinned. S3 creds reuse forte-drop-secrets (the app's UpCloud user has +# s3:* on the drops bucket). PG creds from forte-drop-pg-creds. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: forte-drop-pg-backup + namespace: forte-drop + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: backup +spec: + schedule: "0 2 * * *" # 02:00 UTC daily + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app.kubernetes.io/name: postgresql + app.kubernetes.io/instance: forte-drop + app.kubernetes.io/component: backup + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + volumes: + - name: work + emptyDir: {} + initContainers: + - name: dump + image: postgres:16-alpine + command: + - sh + - -c + - | + set -euo pipefail + TS=$(date -u +%Y%m%dT%H%M%SZ) + echo "dumping to /work/forte-drop-${TS}.sql.gz" + PGPASSWORD="$PGPASSWORD" pg_dump \ + -h forte-drop-postgresql.forte-drop.svc \ + -p 5432 -U "$PGUSER" -d drops \ + --no-owner --no-privileges \ + | gzip -9 > "/work/forte-drop-${TS}.sql.gz" + echo "dump complete: $(ls -lh /work/)" + env: + - name: PGUSER + valueFrom: + secretKeyRef: { name: forte-drop-pg-creds, key: pgusername } + - name: PGPASSWORD + valueFrom: + secretKeyRef: { name: forte-drop-pg-creds, key: pgpassword } + volumeMounts: + - name: work + mountPath: /work + containers: + - name: upload + image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z + command: + - sh + - -c + - | + set -euo pipefail + mc alias set obj "$S3_ENDPOINT" "$S3_KEY" "$S3_SECRET" + mc cp /work/*.sql.gz "obj/${S3_BUCKET}/_pgbackups/" + echo "uploaded. pruning backups older than 30d:" + mc rm --recursive --force --older-than 30d "obj/${S3_BUCKET}/_pgbackups/" || true + echo "backup retention pass complete" + env: + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: { name: forte-drop-secrets, key: S3_ENDPOINT } + - name: S3_BUCKET + value: "drops" + - name: S3_KEY + valueFrom: + secretKeyRef: { name: forte-drop-secrets, key: S3_KEY } + - name: S3_SECRET + valueFrom: + secretKeyRef: { name: forte-drop-secrets, key: S3_SECRET } + volumeMounts: + - name: work + mountPath: /work diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml index 659ba9b..4fa4aa8 100644 --- a/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml +++ b/infra/overlays/upc-dev/forte-drop-postgresql/resources/postgresql.yaml @@ -94,6 +94,8 @@ spec: volumeClaimTemplates: - metadata: name: data + annotations: + argocd.argoproj.io/sync-options: Prune=false,Delete=false spec: accessModes: - ReadWriteOnce diff --git a/infra/overlays/upc-dev/kustomization.yaml b/infra/overlays/upc-dev/kustomization.yaml index f36009e..7ec2160 100644 --- a/infra/overlays/upc-dev/kustomization.yaml +++ b/infra/overlays/upc-dev/kustomization.yaml @@ -5,7 +5,6 @@ resources: - vaultwarden-postgresql - vaultwarden - forte-drop-postgresql -- forte-drop-minio # No patches needed — base already has "upc-dev" paths # upc-dev is the default/base cluster -- 2.49.1 From 94c7924e655e470f3a01ef493c41788d755ac42c Mon Sep 17 00:00:00 2001 From: Sten Date: Fri, 29 May 2026 10:31:09 +0200 Subject: [PATCH 7/7] docs(infra): pg backup & restore runbook for forte-drop Covers: nightly backup mechanism, listing backups, manual trigger, full restore procedure (2-pod mc-download + psql-pipe), verification, object-data note, and a disaster-scenario recovery table. --- .../upc-dev/forte-drop-postgresql/RESTORE.md | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 infra/overlays/upc-dev/forte-drop-postgresql/RESTORE.md diff --git a/infra/overlays/upc-dev/forte-drop-postgresql/RESTORE.md b/infra/overlays/upc-dev/forte-drop-postgresql/RESTORE.md new file mode 100644 index 0000000..028bd6b --- /dev/null +++ b/infra/overlays/upc-dev/forte-drop-postgresql/RESTORE.md @@ -0,0 +1,143 @@ +# forte-drop Postgres — backup & restore runbook + +## What gets backed up + +A CronJob (`forte-drop-pg-backup`, namespace `forte-drop`) runs nightly at **02:00 UTC**: + +1. `pg_dump` of the `drops` database → gzip. +2. Upload to **UpCloud Managed Object Storage**: `s3://drops/_pgbackups/forte-drop-.sql.gz` + (the `_pgbackups/` prefix is collision-proof: app slugs match `/^[a-z0-9][a-z0-9-]{0,62}$/` + and can never start with `_`). +3. Retention: dumps older than **30 days** are pruned. + +S3 creds come from the `forte-drop-secrets` Secret (`S3_ENDPOINT` / `S3_KEY` / `S3_SECRET`). +Postgres creds from `forte-drop-pg-creds` (`pgusername` / `pgpassword`). + +> **Object storage is the durable tier.** App data + DB backups both live in UpCloud +> Managed Object Storage (replicated by UpCloud). The in-cluster Postgres PVC is the +> live working copy; the nightly dump is the recovery point. The PVC carries +> `Prune=false,Delete=false` so ArgoCD never deletes it. + +## Prerequisites + +```bash +export KUBECONFIG=~/Downloads/dev-fd-no-svg1_kubeconfig.yaml +# Confirm the namespace + DB pod are up: +kubectl -n forte-drop get pods -l app.kubernetes.io/name=postgresql +``` + +## List available backups + +```bash +# Run an ephemeral mc pod with the app's S3 creds: +kubectl -n forte-drop run mc-list --rm -it --restart=Never \ + --image=quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z \ + --overrides='{"spec":{"containers":[{"name":"mc","image":"quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z","command":["sh","-c","mc alias set obj \"$S3_ENDPOINT\" \"$S3_KEY\" \"$S3_SECRET\" >/dev/null && mc ls obj/drops/_pgbackups/"],"envFrom":[{"secretRef":{"name":"forte-drop-secrets"}}]}]}}' +``` + +## Manually trigger a backup (before risky changes) + +```bash +kubectl -n forte-drop create job --from=cronjob/forte-drop-pg-backup pg-backup-manual-$(date +%s) +# Watch: +kubectl -n forte-drop get jobs -l app.kubernetes.io/component=backup +kubectl -n forte-drop logs -l app.kubernetes.io/component=backup --tail=40 +``` + +## Restore a dump + +> **Destructive.** This overwrites the live `drops` database. Take a fresh manual +> backup first (above) and confirm with whoever owns the data before proceeding. + +### 1. Pick the dump to restore + +List backups (above), choose `forte-drop-.sql.gz`. + +### 2. Run a restore pod that pulls the dump and pipes it into Postgres + +```bash +DUMP="forte-drop-20260530T020000Z.sql.gz" # <-- set to the chosen file + +kubectl -n forte-drop run pg-restore --rm -it --restart=Never \ + --image=postgres:16-alpine \ + --overrides='{ + "spec": { + "containers": [{ + "name": "restore", + "image": "postgres:16-alpine", + "command": ["sh","-c","set -euo pipefail; \ + apk add --no-cache curl >/dev/null; \ + # download via mc is simpler — use a 2-step instead (see note). \ + echo placeholder"], + "envFrom": [ + {"secretRef":{"name":"forte-drop-pg-creds"}}, + {"secretRef":{"name":"forte-drop-secrets"}} + ] + }] + } + }' +``` + +**Simpler 2-pod approach (recommended — avoids cramming mc + psql in one image):** + +```bash +DUMP="forte-drop-20260530T020000Z.sql.gz" + +# (a) Download the dump from object storage to a local file: +kubectl -n forte-drop run mc-get --rm -it --restart=Never \ + --image=quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z \ + --overrides='{"spec":{"containers":[{"name":"mc","image":"quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z","command":["sh","-c","mc alias set obj \"$S3_ENDPOINT\" \"$S3_KEY\" \"$S3_SECRET\" >/dev/null && mc cat obj/drops/_pgbackups/'"$DUMP"'"],"envFrom":[{"secretRef":{"name":"forte-drop-secrets"}}]}]}}' \ + > /tmp/$DUMP + +# (b) Pipe it into the live Postgres via the service: +gunzip -c /tmp/$DUMP | kubectl -n forte-drop run pg-restore --rm -i --restart=Never \ + --image=postgres:16-alpine \ + --overrides='{"spec":{"containers":[{"name":"psql","image":"postgres:16-alpine","stdin":true,"command":["sh","-c","PGPASSWORD=\"$pgpassword\" psql -h forte-drop-postgresql.forte-drop.svc -U \"$pgusername\" -d drops"],"env":[{"name":"pgusername","valueFrom":{"secretKeyRef":{"name":"forte-drop-pg-creds","key":"pgusername"}}},{"name":"pgpassword","valueFrom":{"secretKeyRef":{"name":"forte-drop-pg-creds","key":"pgpassword"}}}]}]}}' +``` + +> The app's schema is created idempotently on boot (`CREATE TABLE IF NOT EXISTS` + +> `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` in `src/repo/pg.ts`), and `pg_dump` +> output includes the data. For a clean restore into a fresh DB this just works. +> To restore over an existing DB with conflicting rows, drop/recreate the `drops` +> database first (coordinate downtime — scale the web Deployment to 0 during the +> restore so the app isn't writing). + +### 3. Verify + +```bash +kubectl -n forte-drop run pg-check --rm -it --restart=Never \ + --image=postgres:16-alpine \ + --env="PGPASSWORD=$(kubectl -n forte-drop get secret forte-drop-pg-creds -o jsonpath='{.data.pgpassword}' | base64 -d)" \ + --command -- psql -h forte-drop-postgresql.forte-drop.svc -U drops -d drops \ + -c "SELECT count(*) AS drops FROM drops;" -c "SELECT count(*) AS view_hits FROM view_hits;" +``` + +### 4. Bring the app back + +```bash +# If you scaled web to 0 for the restore: +kubectl -n forte-drop scale deploy/forte-drop --replicas=2 +``` + +## Object data (uploaded drop files) + +Drop files live in `s3://drops//...` in the same managed bucket. They are +**not** part of the pg backup (the dump only holds metadata). Object storage is +UpCloud-managed/replicated, so no separate file backup is configured. If a +file-level backup is later required, mirror the bucket to a second bucket/region: + +```bash +mc mirror --overwrite obj/drops/ backup-target/drops-mirror/ +``` + +(Exclude `_pgbackups/` from the app-data mirror if you split them.) + +## Disaster scenarios + +| Scenario | Recovery | +|---|---| +| Postgres pod crash / reschedule | StatefulSet reattaches the PVC; ~1–2 min downtime; no data loss. | +| PVC lost / corrupted | Recreate StatefulSet, restore latest nightly dump (above). Data since last dump is lost. | +| Accidental `drops` table data loss | Restore latest dump; or `pg_restore` a single table from a dump. | +| Namespace deleted | PVC has `Prune=false,Delete=false`; recreate Applications, PVC re-binds, app recovers. Backups in object storage are independent. | +| Object storage bucket lost | UpCloud-managed (replicated). If the IAM key is rotated, update `forte-drop-secrets` (re-seal). | -- 2.49.1