Compare commits
8 Commits
feature/ai
...
4e6a84785a
| Author | SHA1 | Date | |
|---|---|---|---|
| 4e6a84785a | |||
| e0bdaab422 | |||
| c88938adb5 | |||
| d05a16840e | |||
| d7c7242aa1 | |||
| 3bf9fa7837 | |||
| d2596568f2 | |||
| 2a3539350b |
47
.gitea/workflows/ai-review.yaml
Normal file
47
.gitea/workflows/ai-review.yaml
Normal file
@@ -0,0 +1,47 @@
|
||||
name: AI Code Review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [ labeled, synchronize ]
|
||||
|
||||
jobs:
|
||||
ai-review:
|
||||
if: >-
|
||||
(github.event.action == 'synchronized' && contains(toJSON(github.event.pull_request.labels), 'ai-review')) || contains(toJSON(gitea.event.changes.added_labels), 'ai-review')
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
AI_REVIEW_CONFIG_FILE_YAML: ./shared-prompts/iac/.ai-review.yaml
|
||||
# VCS configuration
|
||||
VCS__PROVIDER: GITEA
|
||||
VCS__PIPELINE__OWNER: ${{ github.repository_owner }}
|
||||
VCS__PIPELINE__REPO: ${{ github.event.repository.name }}
|
||||
VCS__PIPELINE__PULL_NUMBER: ${{ github.event.pull_request.number }}
|
||||
VCS__HTTP_CLIENT__API_URL: https://git.forteapps.net/api/v1
|
||||
VCS__HTTP_CLIENT__API_TOKEN: ${{ secrets.AI_REVIEW_TOKEN }}
|
||||
# Review — disable fallback to see real Gitea API errors
|
||||
REVIEW__INLINE_COMMENT_FALLBACK: "false"
|
||||
# LLM configuration
|
||||
LLM__PROVIDER: CLAUDE
|
||||
LLM__META__MODEL: claude-sonnet-4-20250514
|
||||
LLM__META__MAX_TOKENS: "4096"
|
||||
LLM__HTTP_CLIENT__API_URL: https://api.anthropic.com
|
||||
LLM__HTTP_CLIENT__API_TOKEN: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.AI_REVIEW_TOKEN }}
|
||||
|
||||
- name: Run inline review
|
||||
uses: docker://nikitafilonov/ai-review:v0.64.0
|
||||
with:
|
||||
args: ai-review run-inline
|
||||
|
||||
- name: Run summary review
|
||||
uses: docker://nikitafilonov/ai-review:v0.64.0
|
||||
with:
|
||||
args: ai-review run-summary
|
||||
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
[submodule "shared-prompts"]
|
||||
path = shared-prompts
|
||||
url = https://git.forteapps.net/Forte/ai-review-prompts.git
|
||||
@@ -57,17 +57,17 @@ spec:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
mc alias set upcloud "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
|
||||
mc alias set s3 "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
|
||||
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
KEY="gitea-dump-${TIMESTAMP}.zip"
|
||||
echo "Uploading ${KEY}..."
|
||||
mc cp /backup/gitea-dump.zip "upcloud/${S3_BUCKET}/${KEY}" && \
|
||||
mc cp /backup/gitea-dump.zip "s3/${S3_BUCKET}/${KEY}" && \
|
||||
echo "Upload complete."
|
||||
|
||||
# Prune backups older than 7 days
|
||||
echo "Pruning backups older than 7 days..."
|
||||
mc rm --older-than 7d --force "upcloud/${S3_BUCKET}/" 2>&1 || true
|
||||
mc rm --older-than 7d --force "s3/${S3_BUCKET}/" 2>&1 || true
|
||||
echo "Pruning complete."
|
||||
envFrom:
|
||||
- secretRef:
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
- [Kyverno Policies](#kyverno-policies)
|
||||
- [Configuration Reference](#configuration-reference)
|
||||
- [API Endpoints](#api-endpoints)
|
||||
- [Cloud Overlay Pattern](#cloud-overlay-pattern)
|
||||
- [Glossary](#glossary)
|
||||
|
||||
---
|
||||
@@ -92,16 +93,34 @@ launchpad/
|
||||
│ ├── sealedsecrets.yaml
|
||||
│ ├── secrets.yaml
|
||||
│ ├── renovate.yaml
|
||||
│ ├── base/ # ArgoCD Application manifests (Kustomize base)
|
||||
│ │ ├── gitea.yaml
|
||||
│ │ ├── opencost.yaml
|
||||
│ │ ├── traefik-application.yaml
|
||||
│ │ ├── keycloak.yaml
|
||||
│ │ ├── grafana.yaml
|
||||
│ │ └── ...
|
||||
│ ├── overlays/
|
||||
│ │ └── upc-prod/
|
||||
│ │ └── kustomization.yaml # Patches upc-dev → upc-prod valueFile paths
|
||||
│ └── values/
|
||||
│ ├── argocd-values.yaml
|
||||
│ ├── prometheus-values.yaml
|
||||
│ ├── grafana-values.yaml
|
||||
│ ├── loki-values.yaml
|
||||
│ ├── tempo-values.yaml
|
||||
│ ├── gitea-values.yaml
|
||||
│ ├── gitea-actions-values.yaml
|
||||
│ ├── fluent-bit-values.yaml
|
||||
│ └── renovate-values.yaml
|
||||
│ ├── base/ # Cloud-agnostic Helm values
|
||||
│ │ ├── gitea-values.yaml
|
||||
│ │ ├── opencost-values.yaml
|
||||
│ │ ├── prometheus-values.yaml
|
||||
│ │ └── ...
|
||||
│ ├── upc-dev/ # UpCloud dev overlay values
|
||||
│ │ ├── traefik-values.yaml
|
||||
│ │ ├── keycloak-values.yaml
|
||||
│ │ ├── grafana-values.yaml
|
||||
│ │ ├── gitea-values.yaml
|
||||
│ │ └── opencost-values.yaml
|
||||
│ └── upc-prod/ # UpCloud prod overlay values
|
||||
│ ├── traefik-values.yaml
|
||||
│ ├── keycloak-values.yaml
|
||||
│ ├── grafana-values.yaml
|
||||
│ ├── gitea-values.yaml
|
||||
│ └── opencost-values.yaml
|
||||
│
|
||||
├── apps/ # Business applications
|
||||
│ ├── mcp10x.yaml
|
||||
@@ -135,6 +154,15 @@ launchpad/
|
||||
│ ├── mcp10x-credentials-sealed.yaml
|
||||
│ └── musicman-credentials.yaml
|
||||
│
|
||||
├── scripts/ # Operational helper scripts
|
||||
│ ├── gitea-backup.sh # S3 backup helper (list/download)
|
||||
│ ├── gitea-restore.sh
|
||||
│ └── backup/ # Per-cloud backup reference scripts
|
||||
│ ├── s3-minio.sh # S3-compatible (UpCloud, MinIO, Wasabi)
|
||||
│ ├── aws-s3.sh # Native AWS S3
|
||||
│ ├── azure-blob.sh # Azure Blob Storage
|
||||
│ └── gcp-gcs.sh # GCP Cloud Storage
|
||||
│
|
||||
├── private/ # Local-only (Git-ignored)
|
||||
│ ├── *.yaml
|
||||
│ └── *.sh
|
||||
@@ -819,6 +847,8 @@ postgresql:
|
||||
|
||||
**Email Notifications**: Enabled (`ENABLE_NOTIFY_MAIL: true`). SMTP credentials injected via `gitea-smtp-secret` using `additionalConfigFromEnvs` with `GITEA__mailer__USER` / `GITEA__mailer__PASSWD` environment variables.
|
||||
|
||||
**Auto-Watch**: Disabled (`AUTO_WATCH_ON_CHANGES: false`, `AUTO_WATCH_NEW_REPOS: false`). Prevents contributors from being auto-subscribed to repo notifications on push, reducing email noise from CI bots (e.g., ai-review PR comments). Users who were already watching before this change need to manually unwatch or switch to "Only participating".
|
||||
|
||||
**Endpoints**:
|
||||
- Web UI: `https://git.forteapps.net`
|
||||
- SSH: port 22 (ClusterIP)
|
||||
@@ -876,6 +906,84 @@ dind:
|
||||
- Gitea admin panel (`/admin/runners`) — runners show as Online
|
||||
- Create test workflow in `.gitea/workflows/test.yml` — job executes
|
||||
|
||||
### AI Code Review (ai-review)
|
||||
|
||||
**Type**: Gitea Actions workflow (`.gitea/workflows/ai-review.yaml`)
|
||||
**Trigger**: `pull_request` events (`opened`, `synchronize`)
|
||||
**Runner**: `ubuntu-latest` (container: `nikitafilonov/ai-review:latest`)
|
||||
|
||||
**Purpose**: Automated AI-powered code review on pull requests using Claude (Anthropic). Posts inline comments on changed lines and a PR summary comment highlighting infrastructure impact.
|
||||
|
||||
**Architecture**:
|
||||
- Uses [xai-review](https://github.com/nicktechnologies/xai-review) Docker image
|
||||
- Shared configuration and prompts live in the `shared-prompts` Git submodule (→ `Forte/ai-review-prompts`)
|
||||
- Review mode: `ONLY_ADDED_WITH_CONTEXT` — reviews only new/changed lines plus surrounding context (token-efficient)
|
||||
- Agent mode: disabled (one-shot review, no multi-turn reasoning)
|
||||
- LLM: Claude Sonnet (`claude-sonnet-4-20250514`)
|
||||
|
||||
**Shared Prompts Structure** (submodule: `Forte/ai-review-prompts`):
|
||||
```
|
||||
shared-prompts/
|
||||
base/
|
||||
security.md # org-wide security rules (all profiles)
|
||||
iac/
|
||||
.ai-review.yaml # IaC/GitOps profile config
|
||||
inline.md # inline review prompt
|
||||
summary.md # PR summary prompt
|
||||
# future profiles: backend/, frontend/, etc.
|
||||
```
|
||||
|
||||
**Configuration** (`shared-prompts/iac/.ai-review.yaml`):
|
||||
```yaml
|
||||
llm:
|
||||
provider: CLAUDE
|
||||
model: claude-sonnet-4-20250514
|
||||
vcs:
|
||||
provider: GITEA
|
||||
review:
|
||||
mode: ONLY_ADDED_WITH_CONTEXT
|
||||
agent:
|
||||
enabled: false
|
||||
prompt:
|
||||
inline_prompt_files: # concatenated in order
|
||||
- ./shared-prompts/base/security.md
|
||||
- ./shared-prompts/iac/inline.md
|
||||
summary_prompt_files:
|
||||
- ./shared-prompts/iac/summary.md
|
||||
ignore:
|
||||
- "*.sealed.yaml"
|
||||
- "*.lock"
|
||||
- "docs/**"
|
||||
```
|
||||
|
||||
**Custom Prompts** (IaC profile):
|
||||
- `shared-prompts/base/security.md` — org-wide security rules, concatenated before every inline review prompt
|
||||
- `shared-prompts/iac/inline.md` — IaC-specific inline review (YAML, Helm, K8s manifests, shell scripts), max 7 comments
|
||||
- `shared-prompts/iac/summary.md` — PR summary: affected services/namespaces, infrastructure impact, security flags
|
||||
|
||||
**Prompt composition**: ai-review does not support Jinja includes. Instead, list multiple files under `inline_prompt_files` / `summary_prompt_files` — they are concatenated in order with double newlines.
|
||||
|
||||
**Adding a new profile**: Create a new directory (e.g., `backend/`) with its own `.ai-review.yaml`, `inline.md`, and `summary.md`. The `inline_prompt_files` list should include `base/security.md` first, then the profile-specific prompt. Reference it in the consuming repo's workflow: `AI_REVIEW_CONFIG_FILE_YAML=./shared-prompts/backend/.ai-review.yaml`
|
||||
|
||||
**Required Secrets** (configure in Gitea repo or org settings):
|
||||
|
||||
| Secret | Purpose |
|
||||
|--------|---------|
|
||||
| `ANTHROPIC_API_KEY` | Claude API key (from Anthropic console) |
|
||||
| `AI_REVIEW_TOKEN` | Gitea API token with `write:repository` + `read:repository` scopes (use a bot/service account) |
|
||||
|
||||
**Setup Steps**:
|
||||
1. Create a Gitea bot/service account and generate an API token with `write:repository` + `read:repository` scopes
|
||||
2. Add `AI_REVIEW_TOKEN` secret in Gitea repo settings → Actions → Secrets
|
||||
3. Add `ANTHROPIC_API_KEY` secret with your Anthropic API key
|
||||
4. Ensure the `shared-prompts` submodule is initialized (`git submodule update --init`)
|
||||
5. Push the workflow file — it triggers automatically on PR creation/update
|
||||
|
||||
**Verification**:
|
||||
- Open a PR with infrastructure changes → workflow runs → inline comments + summary appear
|
||||
- Check Gitea Actions tab for workflow run status and logs
|
||||
- Monitor Anthropic usage dashboard for token consumption
|
||||
|
||||
### Keycloak Client Registrar
|
||||
|
||||
**Type**: CronJob (deployed via Keycloak Helm chart `extraDeploy`)
|
||||
@@ -1541,6 +1649,79 @@ POST /loki/api/v1/push
|
||||
|
||||
---
|
||||
|
||||
## Cloud Overlay Pattern
|
||||
|
||||
### Overview
|
||||
|
||||
Cloud-specific configuration (StorageClass, LoadBalancer annotations, pricing models, etc.) lives in per-cloud overlay value files, **not** in `base/`. This means adding a new cloud provider (AKS, EKS, GKE) only requires a new overlay directory — no base changes.
|
||||
|
||||
### How It Works
|
||||
|
||||
Each ArgoCD Application uses **multi-source Helm values** with two value files:
|
||||
|
||||
```yaml
|
||||
# infra/base/gitea.yaml (example)
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/infra/values/base/gitea-values.yaml # [0] cloud-agnostic
|
||||
- $values/infra/values/upc-dev/gitea-values.yaml # [1] cloud-specific (default: upc-dev)
|
||||
```
|
||||
|
||||
The `upc-prod` Kustomize overlay patches index `[1]` to swap the cloud-specific file:
|
||||
|
||||
```yaml
|
||||
# infra/overlays/upc-prod/kustomization.yaml
|
||||
- target:
|
||||
kind: Application
|
||||
name: gitea
|
||||
patch: |
|
||||
- op: replace
|
||||
path: /spec/sources/0/helm/valueFiles/1
|
||||
value: $values/infra/values/upc-prod/gitea-values.yaml
|
||||
```
|
||||
|
||||
### Components Using Cloud Overlays
|
||||
|
||||
| Component | Cloud-specific config | Overlay value file |
|
||||
|-----------|----------------------|-------------------|
|
||||
| **Traefik** | LB annotations, proxy protocol IPs | `traefik-values.yaml` |
|
||||
| **Keycloak** | Hostname, TLS settings | `keycloak-values.yaml` |
|
||||
| **Grafana** | Hostname, datasource URLs | `grafana-values.yaml` |
|
||||
| **Gitea** | StorageClass (persistence + PostgreSQL) | `gitea-values.yaml` |
|
||||
| **OpenCost** | Custom pricing model (CPU/RAM/storage rates) | `opencost-values.yaml` |
|
||||
|
||||
### Backup CronJob
|
||||
|
||||
The `gitea-backup` CronJob uses a generic `s3` alias for `minio/mc`. The actual endpoint and credentials come from the `gitea-backup-s3` Sealed Secret, which is per-cloud. Reference scripts for different cloud providers are in `scripts/backup/`:
|
||||
|
||||
| Script | Provider | Tool |
|
||||
|--------|----------|------|
|
||||
| `s3-minio.sh` | S3-compatible (UpCloud, MinIO, Wasabi) | `minio/mc` |
|
||||
| `aws-s3.sh` | AWS S3 | `aws` CLI |
|
||||
| `azure-blob.sh` | Azure Blob Storage | `az` CLI |
|
||||
| `gcp-gcs.sh` | GCP Cloud Storage | `gsutil` |
|
||||
|
||||
### Adding a New Cloud Provider
|
||||
|
||||
To add support for a new cloud (e.g., `aks-dev`):
|
||||
|
||||
1. **Create overlay value directory**: `infra/values/aks-dev/`
|
||||
2. **Add cloud-specific value files** for each component that needs one:
|
||||
- `traefik-values.yaml` — LB annotations, proxy protocol config
|
||||
- `keycloak-values.yaml` — hostname/TLS if different
|
||||
- `grafana-values.yaml` — hostname/datasources if different
|
||||
- `gitea-values.yaml` — `storageClass` for persistence + PostgreSQL
|
||||
- `opencost-values.yaml` — `customPricing` cost model for your cloud
|
||||
3. **Create a Kustomize overlay** (if needed): `infra/overlays/aks-prod/kustomization.yaml`
|
||||
- Patch each Application's `valueFiles[1]` to point to `aks-prod/` files
|
||||
4. **Create a root Application**: `_app-of-apps-aks-dev.yaml` pointing to the overlay
|
||||
5. **Create Sealed Secrets** for the new cluster:
|
||||
- `secrets/aks-dev/` — TLS certs, credentials, backup S3 config
|
||||
6. **Update `gitea-backup-s3` secret** with the new cloud's S3-compatible endpoint
|
||||
7. **Bootstrap**: `kubectl apply -f _app-of-apps-aks-dev.yaml -n argocd`
|
||||
|
||||
---
|
||||
|
||||
## Glossary
|
||||
|
||||
### Terms
|
||||
|
||||
@@ -22,6 +22,7 @@ spec:
|
||||
releaseName: gitea
|
||||
valueFiles:
|
||||
- $values/infra/values/base/gitea-values.yaml
|
||||
- $values/infra/values/upc-dev/gitea-values.yaml
|
||||
|
||||
- repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git
|
||||
targetRevision: HEAD
|
||||
|
||||
@@ -22,6 +22,7 @@ spec:
|
||||
releaseName: opencost
|
||||
valueFiles:
|
||||
- $values/infra/values/base/opencost-values.yaml
|
||||
- $values/infra/values/upc-dev/opencost-values.yaml
|
||||
|
||||
- repoURL: git@github.com:fortedigital/sturdy-adventure.git
|
||||
targetRevision: HEAD
|
||||
|
||||
@@ -31,6 +31,24 @@ patches:
|
||||
path: /spec/sources/0/helm/valueFiles/1
|
||||
value: $values/infra/values/upc-prod/grafana-values.yaml
|
||||
|
||||
# Gitea: swap upc-dev → upc-prod
|
||||
- target:
|
||||
kind: Application
|
||||
name: gitea
|
||||
patch: |
|
||||
- op: replace
|
||||
path: /spec/sources/0/helm/valueFiles/1
|
||||
value: $values/infra/values/upc-prod/gitea-values.yaml
|
||||
|
||||
# OpenCost: swap upc-dev → upc-prod
|
||||
- target:
|
||||
kind: Application
|
||||
name: opencost
|
||||
patch: |
|
||||
- op: replace
|
||||
path: /spec/sources/0/helm/valueFiles/1
|
||||
value: $values/infra/values/upc-prod/opencost-values.yaml
|
||||
|
||||
# Secrets: change path to upc-prod
|
||||
- target:
|
||||
kind: Application
|
||||
|
||||
@@ -29,7 +29,10 @@ gitea:
|
||||
ALLOW_ONLY_EXTERNAL_REGISTRATION: true
|
||||
ENABLE_BASIC_AUTHENTICATION: true
|
||||
ENABLE_PASSWORD_SIGNIN_FORM: false
|
||||
ENABLE_NOTIFY_MAIL: true
|
||||
AUTO_WATCH_ON_CHANGES: false
|
||||
AUTO_WATCH_NEW_REPOS: false
|
||||
ENABLE_NOTIFY_MAIL: false
|
||||
ENABLE_TIMETRACKING: false
|
||||
|
||||
openid:
|
||||
ENABLE_OPENID_SIGNIN: false
|
||||
@@ -127,7 +130,6 @@ persistence:
|
||||
size: 10Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClass: upcloud-block-storage-maxiops
|
||||
|
||||
# -- Recreate strategy to avoid Multi-Attach errors with RWO volumes
|
||||
strategy:
|
||||
@@ -153,7 +155,6 @@ postgresql:
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 8Gi
|
||||
storageClass: upcloud-block-storage-maxiops
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
|
||||
@@ -10,18 +10,6 @@ opencost:
|
||||
serviceName: prometheus-server
|
||||
namespaceName: monitoring
|
||||
port: 80
|
||||
customPricing:
|
||||
enabled: true
|
||||
provider: custom
|
||||
costModel:
|
||||
description: "UpCloud 4-node cluster pricing"
|
||||
CPU: "5.86"
|
||||
RAM: "1.46"
|
||||
GPU: "0"
|
||||
storage: "0.34"
|
||||
zoneNetworkEgress: "0"
|
||||
regionNetworkEgress: "0"
|
||||
internetNetworkEgress: "0"
|
||||
ui:
|
||||
enabled: false
|
||||
service:
|
||||
|
||||
7
infra/values/upc-dev/gitea-values.yaml
Normal file
7
infra/values/upc-dev/gitea-values.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
# UpCloud-specific: block storage class for Gitea + PostgreSQL
|
||||
persistence:
|
||||
storageClass: upcloud-block-storage-maxiops
|
||||
postgresql:
|
||||
primary:
|
||||
persistence:
|
||||
storageClass: upcloud-block-storage-maxiops
|
||||
15
infra/values/upc-dev/opencost-values.yaml
Normal file
15
infra/values/upc-dev/opencost-values.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
# UpCloud-specific: custom pricing model
|
||||
opencost:
|
||||
exporter:
|
||||
customPricing:
|
||||
enabled: true
|
||||
provider: custom
|
||||
costModel:
|
||||
description: "UpCloud 4-node cluster pricing"
|
||||
CPU: "5.86"
|
||||
RAM: "1.46"
|
||||
GPU: "0"
|
||||
storage: "0.34"
|
||||
zoneNetworkEgress: "0"
|
||||
regionNetworkEgress: "0"
|
||||
internetNetworkEgress: "0"
|
||||
7
infra/values/upc-prod/gitea-values.yaml
Normal file
7
infra/values/upc-prod/gitea-values.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
# UpCloud-specific: block storage class for Gitea + PostgreSQL
|
||||
persistence:
|
||||
storageClass: upcloud-block-storage-maxiops
|
||||
postgresql:
|
||||
primary:
|
||||
persistence:
|
||||
storageClass: upcloud-block-storage-maxiops
|
||||
15
infra/values/upc-prod/opencost-values.yaml
Normal file
15
infra/values/upc-prod/opencost-values.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
# UpCloud-specific: custom pricing model
|
||||
opencost:
|
||||
exporter:
|
||||
customPricing:
|
||||
enabled: true
|
||||
provider: custom
|
||||
costModel:
|
||||
description: "UpCloud 4-node cluster pricing"
|
||||
CPU: "5.86"
|
||||
RAM: "1.46"
|
||||
GPU: "0"
|
||||
storage: "0.34"
|
||||
zoneNetworkEgress: "0"
|
||||
regionNetworkEgress: "0"
|
||||
internetNetworkEgress: "0"
|
||||
23
scripts/backup/aws-s3.sh
Normal file
23
scripts/backup/aws-s3.sh
Normal file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# AWS S3 backup upload (native AWS CLI)
|
||||
# Uses: aws cli v2
|
||||
# Env: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION, S3_BUCKET
|
||||
|
||||
BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
KEY="gitea-dump-${TIMESTAMP}.zip"
|
||||
|
||||
echo "Uploading ${KEY}..."
|
||||
aws s3 cp "$BACKUP_FILE" "s3://${S3_BUCKET}/${KEY}"
|
||||
echo "Upload complete."
|
||||
|
||||
# Prune backups older than 7 days
|
||||
echo "Pruning backups older than 7 days..."
|
||||
CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%S 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%S)
|
||||
aws s3api list-objects-v2 --bucket "${S3_BUCKET}" --query "Contents[?LastModified<'${CUTOFF}'].Key" --output text \
|
||||
| tr '\t' '\n' \
|
||||
| while read -r key; do
|
||||
[ -n "$key" ] && aws s3 rm "s3://${S3_BUCKET}/${key}" && echo "Deleted: ${key}"
|
||||
done
|
||||
echo "Pruning complete."
|
||||
36
scripts/backup/azure-blob.sh
Normal file
36
scripts/backup/azure-blob.sh
Normal file
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# Azure Blob Storage backup upload
|
||||
# Uses: az cli
|
||||
# Env: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, AZURE_CONTAINER
|
||||
|
||||
BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
KEY="gitea-dump-${TIMESTAMP}.zip"
|
||||
|
||||
echo "Uploading ${KEY}..."
|
||||
az storage blob upload \
|
||||
--account-name "${AZURE_STORAGE_ACCOUNT}" \
|
||||
--account-key "${AZURE_STORAGE_KEY}" \
|
||||
--container-name "${AZURE_CONTAINER}" \
|
||||
--name "${KEY}" \
|
||||
--file "$BACKUP_FILE" \
|
||||
--overwrite
|
||||
echo "Upload complete."
|
||||
|
||||
# Prune backups older than 7 days
|
||||
echo "Pruning backups older than 7 days..."
|
||||
CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%SZ)
|
||||
az storage blob list \
|
||||
--account-name "${AZURE_STORAGE_ACCOUNT}" \
|
||||
--account-key "${AZURE_STORAGE_KEY}" \
|
||||
--container-name "${AZURE_CONTAINER}" \
|
||||
--query "[?properties.lastModified<'${CUTOFF}'].name" -o tsv \
|
||||
| while read -r name; do
|
||||
[ -n "$name" ] && az storage blob delete \
|
||||
--account-name "${AZURE_STORAGE_ACCOUNT}" \
|
||||
--account-key "${AZURE_STORAGE_KEY}" \
|
||||
--container-name "${AZURE_CONTAINER}" \
|
||||
--name "$name" && echo "Deleted: ${name}"
|
||||
done
|
||||
echo "Pruning complete."
|
||||
26
scripts/backup/gcp-gcs.sh
Normal file
26
scripts/backup/gcp-gcs.sh
Normal file
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# GCP Cloud Storage backup upload
|
||||
# Uses: gsutil (gcloud SDK)
|
||||
# Env: GCS_BUCKET (e.g. gs://my-bucket)
|
||||
|
||||
BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
KEY="gitea-dump-${TIMESTAMP}.zip"
|
||||
|
||||
echo "Uploading ${KEY}..."
|
||||
gsutil cp "$BACKUP_FILE" "${GCS_BUCKET}/${KEY}"
|
||||
echo "Upload complete."
|
||||
|
||||
# Prune backups older than 7 days — GCS lifecycle rules are preferred,
|
||||
# but this works as a manual fallback
|
||||
echo "Pruning backups older than 7 days..."
|
||||
CUTOFF=$(date -d '7 days ago' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -v-7d +%Y-%m-%dT%H:%M:%SZ)
|
||||
gsutil ls -l "${GCS_BUCKET}/" \
|
||||
| grep 'gitea-dump-' \
|
||||
| while read -r size date name; do
|
||||
if [[ "$date" < "$CUTOFF" ]]; then
|
||||
gsutil rm "$name" && echo "Deleted: ${name}"
|
||||
fi
|
||||
done
|
||||
echo "Pruning complete."
|
||||
20
scripts/backup/s3-minio.sh
Normal file
20
scripts/backup/s3-minio.sh
Normal file
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# S3-compatible backup upload (UpCloud Objects, MinIO, Wasabi, etc.)
|
||||
# Uses: minio/mc
|
||||
# Env: S3_ENDPOINT, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, S3_BUCKET
|
||||
|
||||
BACKUP_FILE="${1:?Usage: $0 <backup-file>}"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
KEY="gitea-dump-${TIMESTAMP}.zip"
|
||||
|
||||
mc alias set s3 "${S3_ENDPOINT}" "${AWS_ACCESS_KEY_ID}" "${AWS_SECRET_ACCESS_KEY}"
|
||||
|
||||
echo "Uploading ${KEY}..."
|
||||
mc cp "$BACKUP_FILE" "s3/${S3_BUCKET}/${KEY}"
|
||||
echo "Upload complete."
|
||||
|
||||
# Prune backups older than 7 days
|
||||
echo "Pruning backups older than 7 days..."
|
||||
mc rm --older-than 7d --force "s3/${S3_BUCKET}/" 2>&1 || true
|
||||
echo "Pruning complete."
|
||||
@@ -13,7 +13,7 @@ NAMESPACE="gitea"
|
||||
SECRET="gitea-backup-s3"
|
||||
IMAGE="minio/mc:latest"
|
||||
POD_NAME="gitea-backup-helper"
|
||||
ALIAS_CMD='mc alias set upcloud ${S3_ENDPOINT} ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} > /dev/null'
|
||||
ALIAS_CMD='mc alias set s3 ${S3_ENDPOINT} ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} > /dev/null'
|
||||
|
||||
cleanup() {
|
||||
kubectl -n "$NAMESPACE" delete pod "$POD_NAME" --ignore-not-found --grace-period=0 > /dev/null 2>&1 || true
|
||||
@@ -41,7 +41,7 @@ mc_run() {
|
||||
case "${1:-help}" in
|
||||
list)
|
||||
echo "Listing backups..."
|
||||
mc_run 'mc ls upcloud/${S3_BUCKET}/'
|
||||
mc_run 'mc ls s3/${S3_BUCKET}/'
|
||||
;;
|
||||
|
||||
download)
|
||||
@@ -49,7 +49,7 @@ case "${1:-help}" in
|
||||
|
||||
if [ "$FILE" = "latest" ]; then
|
||||
echo "Finding latest backup..."
|
||||
FILE=$(mc_run 'mc ls upcloud/${S3_BUCKET}/' | sort | tail -1 | awk '{print $NF}' | tr -d '[:space:]')
|
||||
FILE=$(mc_run 'mc ls s3/${S3_BUCKET}/' | sort | tail -1 | awk '{print $NF}' | tr -d '[:space:]')
|
||||
if [ -z "$FILE" ]; then
|
||||
echo "No backups found."
|
||||
exit 1
|
||||
@@ -74,7 +74,7 @@ case "${1:-help}" in
|
||||
kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$POD_NAME" --timeout=60s > /dev/null 2>&1
|
||||
|
||||
echo "Saving to ./$FILE ..."
|
||||
kubectl -n "$NAMESPACE" exec "$POD_NAME" -- sh -c "${ALIAS_CMD} && mc cat upcloud/\${S3_BUCKET}/$FILE" > "./$FILE"
|
||||
kubectl -n "$NAMESPACE" exec "$POD_NAME" -- sh -c "${ALIAS_CMD} && mc cat s3/\${S3_BUCKET}/$FILE" > "./$FILE"
|
||||
cleanup
|
||||
|
||||
echo "Downloaded: ./$FILE"
|
||||
|
||||
1
shared-prompts
Submodule
1
shared-prompts
Submodule
Submodule shared-prompts added at 4ec0c926c7
Reference in New Issue
Block a user