29624e845d
/ test (push) Successful in 10s
The nightly Postgres backup CronJob has been **failing every run** — no backups exist in `s3://drops/_pgbackups/`. **Cause:** the upload container runs as uid 65532 (`runAsNonRoot`). `mc` defaults its config to `$HOME/.mc` = `/.mc` and dies with `mkdir /.mc: permission denied` on the non-writable root fs — before any upload. **Fix:** set `MC_CONFIG_DIR=/work/.mc` (the shared emptyDir, writable via `fsGroup: 65532`). The `pg_dump` initContainer already succeeds; this lets the upload step actually run. Validated: `kubectl kustomize` renders clean; env present on the upload container. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Danijel Simeunovic <danijel.simeunovic@fortedigital.com> Co-authored-by: Sten <sten@Sten-sin-MacBook-Pro.local> Reviewed-on: #23 Reviewed-by: Danijel Simeunovic <danijel.simeunovic@fortedigital.com> Co-authored-by: Jørgen Stensrud <jorgen.stensrud@fortedigital.com> Co-committed-by: Jørgen Stensrud <jorgen.stensrud@fortedigital.com>
100 lines
3.7 KiB
YAML
100 lines
3.7 KiB
YAML
# Nightly logical backup of the forte-drop Postgres → UpCloud Managed Object Storage.
|
|
# Dumps to s3://drops/_pgbackups/ (the `_` prefix is collision-proof: app slugs match
|
|
# /^[a-z0-9][a-z0-9-]{0,62}$/ and can never start with `_`). Retains 30 days.
|
|
#
|
|
# Pod shape: initContainer pg_dump → shared emptyDir → mc upload + retention prune.
|
|
# Both images pinned. S3 creds reuse forte-drop-secrets (the app's UpCloud user has
|
|
# s3:* on the drops bucket). PG creds from forte-drop-pg-creds.
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: forte-drop-pg-backup
|
|
namespace: forte-drop
|
|
labels:
|
|
app.kubernetes.io/name: postgresql
|
|
app.kubernetes.io/instance: forte-drop
|
|
app.kubernetes.io/component: backup
|
|
spec:
|
|
schedule: "0 2 * * *" # 02:00 UTC daily
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: postgresql
|
|
app.kubernetes.io/instance: forte-drop
|
|
app.kubernetes.io/component: backup
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65532
|
|
fsGroup: 65532
|
|
volumes:
|
|
- name: work
|
|
emptyDir: {}
|
|
initContainers:
|
|
- name: dump
|
|
image: postgres:16-alpine
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -euo pipefail
|
|
TS=$(date -u +%Y%m%dT%H%M%SZ)
|
|
echo "dumping to /work/forte-drop-${TS}.sql.gz"
|
|
PGPASSWORD="$PGPASSWORD" pg_dump \
|
|
-h forte-drop-postgresql.forte-drop.svc \
|
|
-p 5432 -U "$PGUSER" -d drops \
|
|
--no-owner --no-privileges \
|
|
| gzip -9 > "/work/forte-drop-${TS}.sql.gz"
|
|
echo "dump complete: $(ls -lh /work/)"
|
|
env:
|
|
- name: PGUSER
|
|
valueFrom:
|
|
secretKeyRef: { name: forte-drop-pg-creds, key: pgusername }
|
|
- name: PGPASSWORD
|
|
valueFrom:
|
|
secretKeyRef: { name: forte-drop-pg-creds, key: pgpassword }
|
|
volumeMounts:
|
|
- name: work
|
|
mountPath: /work
|
|
containers:
|
|
- name: upload
|
|
image: quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z
|
|
command:
|
|
- sh
|
|
- -c
|
|
- |
|
|
set -euo pipefail
|
|
mc alias set obj "$S3_ENDPOINT" "$S3_KEY" "$S3_SECRET"
|
|
mc cp /work/*.sql.gz "obj/${S3_BUCKET}/_pgbackups/"
|
|
echo "uploaded. pruning backups older than 30d:"
|
|
mc rm --recursive --force --older-than 30d "obj/${S3_BUCKET}/_pgbackups/" || true
|
|
echo "backup retention pass complete"
|
|
env:
|
|
# mc writes its config under $MC_CONFIG_DIR; point it at the shared
|
|
# emptyDir (writable by uid 65532 via fsGroup). Without this it tries
|
|
# to mkdir /.mc on the read-only-to-nonroot root fs -> "mkdir /.mc:
|
|
# permission denied" and every run fails before uploading.
|
|
- name: MC_CONFIG_DIR
|
|
value: "/work/.mc"
|
|
- name: S3_ENDPOINT
|
|
valueFrom:
|
|
secretKeyRef: { name: forte-drop-secrets, key: S3_ENDPOINT }
|
|
- name: S3_BUCKET
|
|
value: "drops"
|
|
- name: S3_KEY
|
|
valueFrom:
|
|
secretKeyRef: { name: forte-drop-secrets, key: S3_KEY }
|
|
- name: S3_SECRET
|
|
valueFrom:
|
|
secretKeyRef: { name: forte-drop-secrets, key: S3_SECRET }
|
|
volumeMounts:
|
|
- name: work
|
|
mountPath: /work
|