Forte/launchpad
6
0
Fork 0
Code Issues Pull Requests Actions Packages Wiki Activity
Files
8812b024583c611262bbce8d4e4f330aeac64f5b
launchpad/OPERATIONS-RUNBOOK/index.html
gitea-actions 8812b02458 Deploy docs
2026-04-18 18:39:51 +00:00

4649 lines
245 KiB
HTML
Raw Blame History

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Documentation for the GitOps-managed Kubernetes cluster">
<link rel="prev" href="../DEVELOPER-GUIDE/">
<link rel="next" href="../REFERENCE/">
<link rel="icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>Operations Runbook - K8s Launchpad</title>
<link rel="stylesheet" href="../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#operations-runbook" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href=".." title="K8s Launchpad" class="md-header__button md-logo" aria-label="K8s Launchpad" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
K8s Launchpad
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Operations Runbook
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="indigo" data-md-color-accent="indigo" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://git.forteapps.net/Forte/launchpad" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
Forte/launchpad
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href=".." title="K8s Launchpad" class="md-nav__button md-logo" aria-label="K8s Launchpad" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
K8s Launchpad
</label>
<div class="md-nav__source">
<a href="https://git.forteapps.net/Forte/launchpad" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
Forte/launchpad
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../GITOPS-ARCHITECTURE/" class="md-nav__link">
<span class="md-ellipsis">
GitOps Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../DEVELOPER-GUIDE/" class="md-nav__link">
<span class="md-ellipsis">
Developer Guide
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Operations Runbook
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Operations Runbook
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#table-of-contents" class="md-nav__link">
<span class="md-ellipsis">
Table of Contents
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
<nav class="md-nav" aria-label="Overview">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#operator-prerequisites" class="md-nav__link">
<span class="md-ellipsis">
Operator Prerequisites
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#cluster-bootstrap" class="md-nav__link">
<span class="md-ellipsis">
Cluster Bootstrap
</span>
</a>
<nav class="md-nav" aria-label="Cluster Bootstrap">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#initial-cluster-setup" class="md-nav__link">
<span class="md-ellipsis">
Initial Cluster Setup
</span>
</a>
<nav class="md-nav" aria-label="Initial Cluster Setup">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#prerequisites" class="md-nav__link">
<span class="md-ellipsis">
Prerequisites
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#bootstrap-procedure" class="md-nav__link">
<span class="md-ellipsis">
Bootstrap Procedure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#verify-bootstrap" class="md-nav__link">
<span class="md-ellipsis">
Verify Bootstrap
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#post-bootstrap-steps" class="md-nav__link">
<span class="md-ellipsis">
Post-Bootstrap Steps
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#argocd-repository-access-setup" class="md-nav__link">
<span class="md-ellipsis">
ArgoCD Repository Access Setup
</span>
</a>
<nav class="md-nav" aria-label="ArgoCD Repository Access Setup">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#why-deploy-keys" class="md-nav__link">
<span class="md-ellipsis">
Why Deploy Keys?
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#prerequisites_1" class="md-nav__link">
<span class="md-ellipsis">
Prerequisites
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#setup-procedure" class="md-nav__link">
<span class="md-ellipsis">
Setup Procedure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#testing-repository-access" class="md-nav__link">
<span class="md-ellipsis">
Testing Repository Access
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#security-best-practices" class="md-nav__link">
<span class="md-ellipsis">
Security Best Practices
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#troubleshooting-repository-access" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting Repository Access
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#multiple-repository-setup" class="md-nav__link">
<span class="md-ellipsis">
Multiple Repository Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#converting-https-to-ssh" class="md-nav__link">
<span class="md-ellipsis">
Converting HTTPS to SSH
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#day-to-day-operations" class="md-nav__link">
<span class="md-ellipsis">
Day-to-Day Operations
</span>
</a>
<nav class="md-nav" aria-label="Day-to-Day Operations">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#monitoring-argocd-sync-status" class="md-nav__link">
<span class="md-ellipsis">
Monitoring ArgoCD Sync Status
</span>
</a>
<nav class="md-nav" aria-label="Monitoring ArgoCD Sync Status">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#via-slack" class="md-nav__link">
<span class="md-ellipsis">
Via Slack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#via-cli" class="md-nav__link">
<span class="md-ellipsis">
Via CLI
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#via-argocd-ui" class="md-nav__link">
<span class="md-ellipsis">
Via ArgoCD UI
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#checking-application-health" class="md-nav__link">
<span class="md-ellipsis">
Checking Application Health
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#manual-sync" class="md-nav__link">
<span class="md-ellipsis">
Manual Sync
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pausing-auto-sync" class="md-nav__link">
<span class="md-ellipsis">
Pausing Auto-Sync
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#application-management" class="md-nav__link">
<span class="md-ellipsis">
Application Management
</span>
</a>
<nav class="md-nav" aria-label="Application Management">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#deploying-a-new-application" class="md-nav__link">
<span class="md-ellipsis">
Deploying a New Application
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#removing-an-application" class="md-nav__link">
<span class="md-ellipsis">
Removing an Application
</span>
</a>
<nav class="md-nav" aria-label="Removing an Application">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#safe-removal-procedure" class="md-nav__link">
<span class="md-ellipsis">
Safe Removal Procedure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#removal-without-cascade" class="md-nav__link">
<span class="md-ellipsis">
Removal Without Cascade
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#scaling-applications" class="md-nav__link">
<span class="md-ellipsis">
Scaling Applications
</span>
</a>
<nav class="md-nav" aria-label="Scaling Applications">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#manual-scaling" class="md-nav__link">
<span class="md-ellipsis">
Manual Scaling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#gitops-scaling" class="md-nav__link">
<span class="md-ellipsis">
GitOps Scaling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#auto-scaling-hpa" class="md-nav__link">
<span class="md-ellipsis">
Auto-Scaling (HPA)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#rolling-back-deployments" class="md-nav__link">
<span class="md-ellipsis">
Rolling Back Deployments
</span>
</a>
<nav class="md-nav" aria-label="Rolling Back Deployments">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#option-1-git-revert" class="md-nav__link">
<span class="md-ellipsis">
Option 1: Git Revert
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#option-2-manual-rollback" class="md-nav__link">
<span class="md-ellipsis">
Option 2: Manual Rollback
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#option-3-change-image-tag" class="md-nav__link">
<span class="md-ellipsis">
Option 3: Change Image Tag
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#resource-updates" class="md-nav__link">
<span class="md-ellipsis">
Resource Updates
</span>
</a>
<nav class="md-nav" aria-label="Resource Updates">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#update-resource-limits" class="md-nav__link">
<span class="md-ellipsis">
Update Resource Limits
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#enable-database" class="md-nav__link">
<span class="md-ellipsis">
Enable Database
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#secret-management" class="md-nav__link">
<span class="md-ellipsis">
Secret Management
</span>
</a>
<nav class="md-nav" aria-label="Secret Management">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#creating-secrets" class="md-nav__link">
<span class="md-ellipsis">
Creating Secrets
</span>
</a>
<nav class="md-nav" aria-label="Creating Secrets">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#step-1-get-public-certificate" class="md-nav__link">
<span class="md-ellipsis">
Step 1: Get Public Certificate
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-2-create-plain-secret" class="md-nav__link">
<span class="md-ellipsis">
Step 2: Create Plain Secret
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-3-seal-secret" class="md-nav__link">
<span class="md-ellipsis">
Step 3: Seal Secret
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-4-commit-sealed-secret" class="md-nav__link">
<span class="md-ellipsis">
Step 4: Commit Sealed Secret
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#updating-secrets" class="md-nav__link">
<span class="md-ellipsis">
Updating Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#viewing-secrets-unsealed" class="md-nav__link">
<span class="md-ellipsis">
Viewing Secrets (Unsealed)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#secret-cloning-kyverno" class="md-nav__link">
<span class="md-ellipsis">
Secret Cloning (Kyverno)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#authentication-secrets" class="md-nav__link">
<span class="md-ellipsis">
Authentication Secrets
</span>
</a>
<nav class="md-nav" aria-label="Authentication Secrets">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#token-mode-secrets" class="md-nav__link">
<span class="md-ellipsis">
Token Mode Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#oidc-mode-secrets" class="md-nav__link">
<span class="md-ellipsis">
OIDC Mode Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#rotating-authentication-secrets" class="md-nav__link">
<span class="md-ellipsis">
Rotating Authentication Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#viewing-authentication-secrets" class="md-nav__link">
<span class="md-ellipsis">
Viewing Authentication Secrets
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#monitoring-alerting" class="md-nav__link">
<span class="md-ellipsis">
Monitoring &amp; Alerting
</span>
</a>
<nav class="md-nav" aria-label="Monitoring &amp; Alerting">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#prometheus-metrics" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Metrics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#grafana-dashboards" class="md-nav__link">
<span class="md-ellipsis">
Grafana Dashboards
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#loki-logs" class="md-nav__link">
<span class="md-ellipsis">
Loki Logs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#tempo-traces" class="md-nav__link">
<span class="md-ellipsis">
Tempo Traces
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#fluent-bit-log-shipping" class="md-nav__link">
<span class="md-ellipsis">
Fluent-Bit Log Shipping
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#trivy-vulnerability-scanning" class="md-nav__link">
<span class="md-ellipsis">
Trivy Vulnerability Scanning
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#slack-notifications" class="md-nav__link">
<span class="md-ellipsis">
Slack Notifications
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#troubleshooting" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
<nav class="md-nav" aria-label="Troubleshooting">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#application-wont-sync" class="md-nav__link">
<span class="md-ellipsis">
Application Won't Sync
</span>
</a>
<nav class="md-nav" aria-label="Application Won&#39;t Sync">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#check-application-status" class="md-nav__link">
<span class="md-ellipsis">
Check Application Status
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#common-issues" class="md-nav__link">
<span class="md-ellipsis">
Common Issues
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#pod-crashes" class="md-nav__link">
<span class="md-ellipsis">
Pod Crashes
</span>
</a>
<nav class="md-nav" aria-label="Pod Crashes">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#crashloopbackoff" class="md-nav__link">
<span class="md-ellipsis">
CrashLoopBackOff
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#imagepullbackoff" class="md-nav__link">
<span class="md-ellipsis">
ImagePullBackOff
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pending" class="md-nav__link">
<span class="md-ellipsis">
Pending
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#ingress-tls-issues" class="md-nav__link">
<span class="md-ellipsis">
Ingress / TLS Issues
</span>
</a>
<nav class="md-nav" aria-label="Ingress / TLS Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#application-not-accessible" class="md-nav__link">
<span class="md-ellipsis">
Application Not Accessible
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#certificate-issues" class="md-nav__link">
<span class="md-ellipsis">
Certificate Issues
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#database-issues" class="md-nav__link">
<span class="md-ellipsis">
Database Issues
</span>
</a>
<nav class="md-nav" aria-label="Database Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#postgresql-wont-start" class="md-nav__link">
<span class="md-ellipsis">
PostgreSQL Won't Start
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#data-persistence" class="md-nav__link">
<span class="md-ellipsis">
Data Persistence
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#kyverno-policy-issues" class="md-nav__link">
<span class="md-ellipsis">
Kyverno Policy Issues
</span>
</a>
<nav class="md-nav" aria-label="Kyverno Policy Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#policy-violations" class="md-nav__link">
<span class="md-ellipsis">
Policy Violations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#secret-not-cloned" class="md-nav__link">
<span class="md-ellipsis">
Secret Not Cloned
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#argocd-issues" class="md-nav__link">
<span class="md-ellipsis">
ArgoCD Issues
</span>
</a>
<nav class="md-nav" aria-label="ArgoCD Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#argocd-ui-not-accessible" class="md-nav__link">
<span class="md-ellipsis">
ArgoCD UI Not Accessible
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#sync-takes-too-long" class="md-nav__link">
<span class="md-ellipsis">
Sync Takes Too Long
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#disaster-recovery" class="md-nav__link">
<span class="md-ellipsis">
Disaster Recovery
</span>
</a>
<nav class="md-nav" aria-label="Disaster Recovery">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#backup-strategy" class="md-nav__link">
<span class="md-ellipsis">
Backup Strategy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cluster-rebuild" class="md-nav__link">
<span class="md-ellipsis">
Cluster Rebuild
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#future-backup-plan" class="md-nav__link">
<span class="md-ellipsis">
Future Backup Plan
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#maintenance-procedures" class="md-nav__link">
<span class="md-ellipsis">
Maintenance Procedures
</span>
</a>
<nav class="md-nav" aria-label="Maintenance Procedures">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#upgrading-argocd" class="md-nav__link">
<span class="md-ellipsis">
Upgrading ArgoCD
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#upgrading-kubernetes-version" class="md-nav__link">
<span class="md-ellipsis">
Upgrading Kubernetes Version
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#rotating-tls-certificates" class="md-nav__link">
<span class="md-ellipsis">
Rotating TLS Certificates
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cleaning-up-old-resources" class="md-nav__link">
<span class="md-ellipsis">
Cleaning Up Old Resources
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dns-management" class="md-nav__link">
<span class="md-ellipsis">
DNS Management
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#monitoring-resource-usage" class="md-nav__link">
<span class="md-ellipsis">
Monitoring Resource Usage
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#advanced-operations" class="md-nav__link">
<span class="md-ellipsis">
Advanced Operations
</span>
</a>
<nav class="md-nav" aria-label="Advanced Operations">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#adding-a-new-infrastructure-component" class="md-nav__link">
<span class="md-ellipsis">
Adding a New Infrastructure Component
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#multi-cluster-setup" class="md-nav__link">
<span class="md-ellipsis">
Multi-Cluster Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#blue-green-deployments" class="md-nav__link">
<span class="md-ellipsis">
Blue-Green Deployments
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#emergency-procedures" class="md-nav__link">
<span class="md-ellipsis">
Emergency Procedures
</span>
</a>
<nav class="md-nav" aria-label="Emergency Procedures">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#emergency-rollback" class="md-nav__link">
<span class="md-ellipsis">
Emergency Rollback
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#emergency-scale-down" class="md-nav__link">
<span class="md-ellipsis">
Emergency Scale Down
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#emergency-application-removal" class="md-nav__link">
<span class="md-ellipsis">
Emergency Application Removal
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#useful-scripts" class="md-nav__link">
<span class="md-ellipsis">
Useful Scripts
</span>
</a>
<nav class="md-nav" aria-label="Useful Scripts">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#sync-all-applications" class="md-nav__link">
<span class="md-ellipsis">
Sync All Applications
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#check-all-applications-health" class="md-nav__link">
<span class="md-ellipsis">
Check All Applications Health
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#seal-secret-helper" class="md-nav__link">
<span class="md-ellipsis">
Seal Secret Helper
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#checklist-templates" class="md-nav__link">
<span class="md-ellipsis">
Checklist Templates
</span>
</a>
<nav class="md-nav" aria-label="Checklist Templates">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#new-application-deployment-checklist" class="md-nav__link">
<span class="md-ellipsis">
New Application Deployment Checklist
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#incident-response-checklist" class="md-nav__link">
<span class="md-ellipsis">
Incident Response Checklist
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../REFERENCE/" class="md-nav__link">
<span class="md-ellipsis">
Technical Reference
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#table-of-contents" class="md-nav__link">
<span class="md-ellipsis">
Table of Contents
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#overview" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
<nav class="md-nav" aria-label="Overview">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#operator-prerequisites" class="md-nav__link">
<span class="md-ellipsis">
Operator Prerequisites
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#cluster-bootstrap" class="md-nav__link">
<span class="md-ellipsis">
Cluster Bootstrap
</span>
</a>
<nav class="md-nav" aria-label="Cluster Bootstrap">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#initial-cluster-setup" class="md-nav__link">
<span class="md-ellipsis">
Initial Cluster Setup
</span>
</a>
<nav class="md-nav" aria-label="Initial Cluster Setup">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#prerequisites" class="md-nav__link">
<span class="md-ellipsis">
Prerequisites
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#bootstrap-procedure" class="md-nav__link">
<span class="md-ellipsis">
Bootstrap Procedure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#verify-bootstrap" class="md-nav__link">
<span class="md-ellipsis">
Verify Bootstrap
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#post-bootstrap-steps" class="md-nav__link">
<span class="md-ellipsis">
Post-Bootstrap Steps
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#argocd-repository-access-setup" class="md-nav__link">
<span class="md-ellipsis">
ArgoCD Repository Access Setup
</span>
</a>
<nav class="md-nav" aria-label="ArgoCD Repository Access Setup">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#why-deploy-keys" class="md-nav__link">
<span class="md-ellipsis">
Why Deploy Keys?
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#prerequisites_1" class="md-nav__link">
<span class="md-ellipsis">
Prerequisites
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#setup-procedure" class="md-nav__link">
<span class="md-ellipsis">
Setup Procedure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#testing-repository-access" class="md-nav__link">
<span class="md-ellipsis">
Testing Repository Access
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#security-best-practices" class="md-nav__link">
<span class="md-ellipsis">
Security Best Practices
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#troubleshooting-repository-access" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting Repository Access
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#multiple-repository-setup" class="md-nav__link">
<span class="md-ellipsis">
Multiple Repository Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#converting-https-to-ssh" class="md-nav__link">
<span class="md-ellipsis">
Converting HTTPS to SSH
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#day-to-day-operations" class="md-nav__link">
<span class="md-ellipsis">
Day-to-Day Operations
</span>
</a>
<nav class="md-nav" aria-label="Day-to-Day Operations">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#monitoring-argocd-sync-status" class="md-nav__link">
<span class="md-ellipsis">
Monitoring ArgoCD Sync Status
</span>
</a>
<nav class="md-nav" aria-label="Monitoring ArgoCD Sync Status">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#via-slack" class="md-nav__link">
<span class="md-ellipsis">
Via Slack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#via-cli" class="md-nav__link">
<span class="md-ellipsis">
Via CLI
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#via-argocd-ui" class="md-nav__link">
<span class="md-ellipsis">
Via ArgoCD UI
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#checking-application-health" class="md-nav__link">
<span class="md-ellipsis">
Checking Application Health
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#manual-sync" class="md-nav__link">
<span class="md-ellipsis">
Manual Sync
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pausing-auto-sync" class="md-nav__link">
<span class="md-ellipsis">
Pausing Auto-Sync
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#application-management" class="md-nav__link">
<span class="md-ellipsis">
Application Management
</span>
</a>
<nav class="md-nav" aria-label="Application Management">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#deploying-a-new-application" class="md-nav__link">
<span class="md-ellipsis">
Deploying a New Application
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#removing-an-application" class="md-nav__link">
<span class="md-ellipsis">
Removing an Application
</span>
</a>
<nav class="md-nav" aria-label="Removing an Application">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#safe-removal-procedure" class="md-nav__link">
<span class="md-ellipsis">
Safe Removal Procedure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#removal-without-cascade" class="md-nav__link">
<span class="md-ellipsis">
Removal Without Cascade
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#scaling-applications" class="md-nav__link">
<span class="md-ellipsis">
Scaling Applications
</span>
</a>
<nav class="md-nav" aria-label="Scaling Applications">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#manual-scaling" class="md-nav__link">
<span class="md-ellipsis">
Manual Scaling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#gitops-scaling" class="md-nav__link">
<span class="md-ellipsis">
GitOps Scaling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#auto-scaling-hpa" class="md-nav__link">
<span class="md-ellipsis">
Auto-Scaling (HPA)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#rolling-back-deployments" class="md-nav__link">
<span class="md-ellipsis">
Rolling Back Deployments
</span>
</a>
<nav class="md-nav" aria-label="Rolling Back Deployments">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#option-1-git-revert" class="md-nav__link">
<span class="md-ellipsis">
Option 1: Git Revert
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#option-2-manual-rollback" class="md-nav__link">
<span class="md-ellipsis">
Option 2: Manual Rollback
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#option-3-change-image-tag" class="md-nav__link">
<span class="md-ellipsis">
Option 3: Change Image Tag
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#resource-updates" class="md-nav__link">
<span class="md-ellipsis">
Resource Updates
</span>
</a>
<nav class="md-nav" aria-label="Resource Updates">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#update-resource-limits" class="md-nav__link">
<span class="md-ellipsis">
Update Resource Limits
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#enable-database" class="md-nav__link">
<span class="md-ellipsis">
Enable Database
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#secret-management" class="md-nav__link">
<span class="md-ellipsis">
Secret Management
</span>
</a>
<nav class="md-nav" aria-label="Secret Management">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#creating-secrets" class="md-nav__link">
<span class="md-ellipsis">
Creating Secrets
</span>
</a>
<nav class="md-nav" aria-label="Creating Secrets">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#step-1-get-public-certificate" class="md-nav__link">
<span class="md-ellipsis">
Step 1: Get Public Certificate
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-2-create-plain-secret" class="md-nav__link">
<span class="md-ellipsis">
Step 2: Create Plain Secret
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-3-seal-secret" class="md-nav__link">
<span class="md-ellipsis">
Step 3: Seal Secret
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-4-commit-sealed-secret" class="md-nav__link">
<span class="md-ellipsis">
Step 4: Commit Sealed Secret
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#updating-secrets" class="md-nav__link">
<span class="md-ellipsis">
Updating Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#viewing-secrets-unsealed" class="md-nav__link">
<span class="md-ellipsis">
Viewing Secrets (Unsealed)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#secret-cloning-kyverno" class="md-nav__link">
<span class="md-ellipsis">
Secret Cloning (Kyverno)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#authentication-secrets" class="md-nav__link">
<span class="md-ellipsis">
Authentication Secrets
</span>
</a>
<nav class="md-nav" aria-label="Authentication Secrets">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#token-mode-secrets" class="md-nav__link">
<span class="md-ellipsis">
Token Mode Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#oidc-mode-secrets" class="md-nav__link">
<span class="md-ellipsis">
OIDC Mode Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#rotating-authentication-secrets" class="md-nav__link">
<span class="md-ellipsis">
Rotating Authentication Secrets
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#viewing-authentication-secrets" class="md-nav__link">
<span class="md-ellipsis">
Viewing Authentication Secrets
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#monitoring-alerting" class="md-nav__link">
<span class="md-ellipsis">
Monitoring &amp; Alerting
</span>
</a>
<nav class="md-nav" aria-label="Monitoring &amp; Alerting">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#prometheus-metrics" class="md-nav__link">
<span class="md-ellipsis">
Prometheus Metrics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#grafana-dashboards" class="md-nav__link">
<span class="md-ellipsis">
Grafana Dashboards
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#loki-logs" class="md-nav__link">
<span class="md-ellipsis">
Loki Logs
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#tempo-traces" class="md-nav__link">
<span class="md-ellipsis">
Tempo Traces
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#fluent-bit-log-shipping" class="md-nav__link">
<span class="md-ellipsis">
Fluent-Bit Log Shipping
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#trivy-vulnerability-scanning" class="md-nav__link">
<span class="md-ellipsis">
Trivy Vulnerability Scanning
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#slack-notifications" class="md-nav__link">
<span class="md-ellipsis">
Slack Notifications
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#troubleshooting" class="md-nav__link">
<span class="md-ellipsis">
Troubleshooting
</span>
</a>
<nav class="md-nav" aria-label="Troubleshooting">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#application-wont-sync" class="md-nav__link">
<span class="md-ellipsis">
Application Won't Sync
</span>
</a>
<nav class="md-nav" aria-label="Application Won&#39;t Sync">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#check-application-status" class="md-nav__link">
<span class="md-ellipsis">
Check Application Status
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#common-issues" class="md-nav__link">
<span class="md-ellipsis">
Common Issues
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#pod-crashes" class="md-nav__link">
<span class="md-ellipsis">
Pod Crashes
</span>
</a>
<nav class="md-nav" aria-label="Pod Crashes">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#crashloopbackoff" class="md-nav__link">
<span class="md-ellipsis">
CrashLoopBackOff
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#imagepullbackoff" class="md-nav__link">
<span class="md-ellipsis">
ImagePullBackOff
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pending" class="md-nav__link">
<span class="md-ellipsis">
Pending
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#ingress-tls-issues" class="md-nav__link">
<span class="md-ellipsis">
Ingress / TLS Issues
</span>
</a>
<nav class="md-nav" aria-label="Ingress / TLS Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#application-not-accessible" class="md-nav__link">
<span class="md-ellipsis">
Application Not Accessible
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#certificate-issues" class="md-nav__link">
<span class="md-ellipsis">
Certificate Issues
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#database-issues" class="md-nav__link">
<span class="md-ellipsis">
Database Issues
</span>
</a>
<nav class="md-nav" aria-label="Database Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#postgresql-wont-start" class="md-nav__link">
<span class="md-ellipsis">
PostgreSQL Won't Start
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#data-persistence" class="md-nav__link">
<span class="md-ellipsis">
Data Persistence
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#kyverno-policy-issues" class="md-nav__link">
<span class="md-ellipsis">
Kyverno Policy Issues
</span>
</a>
<nav class="md-nav" aria-label="Kyverno Policy Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#policy-violations" class="md-nav__link">
<span class="md-ellipsis">
Policy Violations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#secret-not-cloned" class="md-nav__link">
<span class="md-ellipsis">
Secret Not Cloned
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#argocd-issues" class="md-nav__link">
<span class="md-ellipsis">
ArgoCD Issues
</span>
</a>
<nav class="md-nav" aria-label="ArgoCD Issues">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#argocd-ui-not-accessible" class="md-nav__link">
<span class="md-ellipsis">
ArgoCD UI Not Accessible
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#sync-takes-too-long" class="md-nav__link">
<span class="md-ellipsis">
Sync Takes Too Long
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#disaster-recovery" class="md-nav__link">
<span class="md-ellipsis">
Disaster Recovery
</span>
</a>
<nav class="md-nav" aria-label="Disaster Recovery">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#backup-strategy" class="md-nav__link">
<span class="md-ellipsis">
Backup Strategy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cluster-rebuild" class="md-nav__link">
<span class="md-ellipsis">
Cluster Rebuild
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#future-backup-plan" class="md-nav__link">
<span class="md-ellipsis">
Future Backup Plan
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#maintenance-procedures" class="md-nav__link">
<span class="md-ellipsis">
Maintenance Procedures
</span>
</a>
<nav class="md-nav" aria-label="Maintenance Procedures">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#upgrading-argocd" class="md-nav__link">
<span class="md-ellipsis">
Upgrading ArgoCD
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#upgrading-kubernetes-version" class="md-nav__link">
<span class="md-ellipsis">
Upgrading Kubernetes Version
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#rotating-tls-certificates" class="md-nav__link">
<span class="md-ellipsis">
Rotating TLS Certificates
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#cleaning-up-old-resources" class="md-nav__link">
<span class="md-ellipsis">
Cleaning Up Old Resources
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dns-management" class="md-nav__link">
<span class="md-ellipsis">
DNS Management
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#monitoring-resource-usage" class="md-nav__link">
<span class="md-ellipsis">
Monitoring Resource Usage
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#advanced-operations" class="md-nav__link">
<span class="md-ellipsis">
Advanced Operations
</span>
</a>
<nav class="md-nav" aria-label="Advanced Operations">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#adding-a-new-infrastructure-component" class="md-nav__link">
<span class="md-ellipsis">
Adding a New Infrastructure Component
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#multi-cluster-setup" class="md-nav__link">
<span class="md-ellipsis">
Multi-Cluster Setup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#blue-green-deployments" class="md-nav__link">
<span class="md-ellipsis">
Blue-Green Deployments
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#emergency-procedures" class="md-nav__link">
<span class="md-ellipsis">
Emergency Procedures
</span>
</a>
<nav class="md-nav" aria-label="Emergency Procedures">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#emergency-rollback" class="md-nav__link">
<span class="md-ellipsis">
Emergency Rollback
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#emergency-scale-down" class="md-nav__link">
<span class="md-ellipsis">
Emergency Scale Down
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#emergency-application-removal" class="md-nav__link">
<span class="md-ellipsis">
Emergency Application Removal
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#useful-scripts" class="md-nav__link">
<span class="md-ellipsis">
Useful Scripts
</span>
</a>
<nav class="md-nav" aria-label="Useful Scripts">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#sync-all-applications" class="md-nav__link">
<span class="md-ellipsis">
Sync All Applications
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#check-all-applications-health" class="md-nav__link">
<span class="md-ellipsis">
Check All Applications Health
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#seal-secret-helper" class="md-nav__link">
<span class="md-ellipsis">
Seal Secret Helper
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#checklist-templates" class="md-nav__link">
<span class="md-ellipsis">
Checklist Templates
</span>
</a>
<nav class="md-nav" aria-label="Checklist Templates">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#new-application-deployment-checklist" class="md-nav__link">
<span class="md-ellipsis">
New Application Deployment Checklist
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#incident-response-checklist" class="md-nav__link">
<span class="md-ellipsis">
Incident Response Checklist
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="operations-runbook">Operations Runbook<a class="headerlink" href="#operations-runbook" title="Permanent link">&para;</a></h1>
<h2 id="table-of-contents">Table of Contents<a class="headerlink" href="#table-of-contents" title="Permanent link">&para;</a></h2>
<ul>
<li><a href="#overview">Overview</a></li>
<li><a href="#cluster-bootstrap">Cluster Bootstrap</a></li>
<li><a href="#initial-cluster-setup">Initial Cluster Setup</a></li>
<li><a href="#argocd-repository-access-setup">ArgoCD Repository Access Setup</a></li>
<li><a href="#day-to-day-operations">Day-to-Day Operations</a></li>
<li><a href="#application-management">Application Management</a></li>
<li><a href="#secret-management">Secret Management</a></li>
<li><a href="#monitoring--alerting">Monitoring &amp; Alerting</a></li>
<li><a href="#troubleshooting">Troubleshooting</a></li>
<li><a href="#disaster-recovery">Disaster Recovery</a></li>
<li><a href="#maintenance-procedures">Maintenance Procedures</a></li>
</ul>
<hr />
<h2 id="overview">Overview<a class="headerlink" href="#overview" title="Permanent link">&para;</a></h2>
<p>This runbook provides operational procedures for maintaining the Kubernetes cluster and managing applications. It's intended for platform engineers and operators with full cluster access.</p>
<h3 id="operator-prerequisites">Operator Prerequisites<a class="headerlink" href="#operator-prerequisites" title="Permanent link">&para;</a></h3>
<ul>
<li>✅ Full kubectl access to cluster</li>
<li>✅ Write access to all Git repositories</li>
<li>✅ ArgoCD UI access</li>
<li>✅ Slack notifications configured</li>
<li>✅ Understanding of Kubernetes concepts</li>
</ul>
<hr />
<h2 id="cluster-bootstrap">Cluster Bootstrap<a class="headerlink" href="#cluster-bootstrap" title="Permanent link">&para;</a></h2>
<h3 id="initial-cluster-setup">Initial Cluster Setup<a class="headerlink" href="#initial-cluster-setup" title="Permanent link">&para;</a></h3>
<p>Bootstrap a new cluster from scratch:</p>
<h4 id="prerequisites">Prerequisites<a class="headerlink" href="#prerequisites" title="Permanent link">&para;</a></h4>
<ol>
<li><strong>Kubernetes cluster running</strong> (UpCloud or any K8s cluster)</li>
<li><strong>kubectl configured</strong> with admin access</li>
<li><strong>Repositories cloned</strong> locally</li>
</ol>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="c1"># Verify cluster access</span>
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>kubectl<span class="w"> </span>cluster-info
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a>kubectl<span class="w"> </span>get<span class="w"> </span>nodes
</code></pre></div>
<h4 id="bootstrap-procedure">Bootstrap Procedure<a class="headerlink" href="#bootstrap-procedure" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="c1"># 1. Clone config repository</span>
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>git<span class="w"> </span>clone<span class="w"> </span>https://git.forteapps.net/Forte/launchpad
<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="nb">cd</span><span class="w"> </span>launchpad
<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a>
<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="c1"># 2. Set cluster name (optional)</span>
<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="nb">export</span><span class="w"> </span><span class="nv">CLUSTER_NAME</span><span class="o">=</span><span class="s2">&quot;prod-cluster-01&quot;</span>
<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a>
<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="c1"># 3. Run bootstrap script</span>
<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a>./bootstrap.sh
</code></pre></div>
<p><strong>What Happens:</strong>
1. ✅ Installs ArgoCD via Helm
2. ✅ Configures ArgoCD with custom values
3. ✅ Applies root App-of-Apps manifest
4. ✅ ArgoCD automatically syncs all applications
5. ✅ Infrastructure and apps deploy in waves</p>
<h4 id="verify-bootstrap">Verify Bootstrap<a class="headerlink" href="#verify-bootstrap" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="c1"># Wait for ArgoCD to be ready</span>
<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a>kubectl<span class="w"> </span><span class="nb">wait</span><span class="w"> </span>--for<span class="o">=</span><span class="nv">condition</span><span class="o">=</span>available<span class="w"> </span>--timeout<span class="o">=</span>300s<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w"> </span>deployment/argocd-server<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a>
<a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="c1"># Check ArgoCD applications</span>
<a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a>
<a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="c1"># Expected output: infrastructure-apps, enterprise-apps, and all child apps</span>
</code></pre></div>
<h4 id="post-bootstrap-steps">Post-Bootstrap Steps<a class="headerlink" href="#post-bootstrap-steps" title="Permanent link">&para;</a></h4>
<ol>
<li><strong>Configure DNS</strong> for ingress domains:</li>
<li><code>argocd.127.0.0.1.nip.io</code> (local dev)</li>
<li>
<p><code>*.forteapps.net</code> (production)</p>
</li>
<li>
<p><strong>Verify Let's Encrypt certificates</strong>:
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>kubectl<span class="w"> </span>get<span class="w"> </span>certificate<span class="w"> </span>--all-namespaces
<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>clusterissuer
</code></pre></div></p>
</li>
<li>
<p><strong>Check Kyverno policies</strong>:
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a>kubectl<span class="w"> </span>get<span class="w"> </span>clusterpolicy
</code></pre></div></p>
</li>
<li>
<p><strong>Verify monitoring stack</strong>:
<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pods<span class="w"> </span>-n<span class="w"> </span>monitoring
</code></pre></div></p>
</li>
<li>
<p><strong>Test Slack notifications</strong> by triggering a sync</p>
</li>
</ol>
<h3 id="argocd-repository-access-setup">ArgoCD Repository Access Setup<a class="headerlink" href="#argocd-repository-access-setup" title="Permanent link">&para;</a></h3>
<p>ArgoCD needs SSH access to private Git repositories to pull manifests and Helm values. This section covers setting up deploy keys for GitHub repositories.</p>
<h4 id="why-deploy-keys">Why Deploy Keys?<a class="headerlink" href="#why-deploy-keys" title="Permanent link">&para;</a></h4>
<ul>
<li><strong>Read-only access</strong>: Deploy keys provide secure, read-only access to repositories</li>
<li><strong>No user credentials</strong>: No need to share personal SSH keys or tokens</li>
<li><strong>Repository-specific</strong>: Each repository gets its own key for better security</li>
<li><strong>Revocable</strong>: Easy to revoke access without affecting other repositories</li>
</ul>
<h4 id="prerequisites_1">Prerequisites<a class="headerlink" href="#prerequisites_1" title="Permanent link">&para;</a></h4>
<ul>
<li>kubectl access to the cluster</li>
<li>Write access to the GitHub repository</li>
<li>ArgoCD installed and running</li>
</ul>
<h4 id="setup-procedure">Setup Procedure<a class="headerlink" href="#setup-procedure" title="Permanent link">&para;</a></h4>
<p><strong>Step 1: Generate SSH Key Pair</strong></p>
<p>Generate a dedicated SSH key for ArgoCD without a passphrase (required for automated access):</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a><span class="c1"># Generate ED25519 key (recommended - smaller and more secure)</span>
<a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a>ssh-keygen<span class="w"> </span>-t<span class="w"> </span>ed25519<span class="w"> </span>-C<span class="w"> </span><span class="s2">&quot;argocd-deploy-key-launchpad&quot;</span><span class="w"> </span>-f<span class="w"> </span>argocd-deploy-key<span class="w"> </span>-N<span class="w"> </span><span class="s2">&quot;&quot;</span>
<a id="__codelineno-6-3" name="__codelineno-6-3" href="#__codelineno-6-3"></a>
<a id="__codelineno-6-4" name="__codelineno-6-4" href="#__codelineno-6-4"></a><span class="c1"># Or RSA key if ED25519 is not supported</span>
<a id="__codelineno-6-5" name="__codelineno-6-5" href="#__codelineno-6-5"></a>ssh-keygen<span class="w"> </span>-t<span class="w"> </span>rsa<span class="w"> </span>-b<span class="w"> </span><span class="m">4096</span><span class="w"> </span>-C<span class="w"> </span><span class="s2">&quot;argocd-deploy-key-launchpad&quot;</span><span class="w"> </span>-f<span class="w"> </span>argocd-deploy-key<span class="w"> </span>-N<span class="w"> </span><span class="s2">&quot;&quot;</span>
</code></pre></div>
<p>This creates two files:
- <code>argocd-deploy-key</code> - Private key (keep secret)
- <code>argocd-deploy-key.pub</code> - Public key (add to GitHub)</p>
<p><strong>Step 2: Add Public Key to GitHub</strong></p>
<ol>
<li>
<p>Copy the public key:
<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>cat<span class="w"> </span>argocd-deploy-key.pub
</code></pre></div></p>
</li>
<li>
<p>Go to GitHub repository settings:</p>
</li>
<li>Navigate to: <code>https://git.forteapps.net/Forte/launchpad/settings/keys</code></li>
<li>
<p>Or: Repository → Settings → Deploy keys</p>
</li>
<li>
<p>Click <strong>"Add deploy key"</strong></p>
</li>
<li>Title: <code>ArgoCD Production Cluster</code></li>
<li>Key: Paste the public key content</li>
<li>☐ Allow write access (leave unchecked - read-only is sufficient)</li>
<li>
<p>Click <strong>"Add key"</strong></p>
</li>
<li>
<p>Repeat for the <code>helm-values</code> repository if it's private:
<div class="highlight"><pre><span></span><code><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a><span class="c1"># Generate separate key for helm-values repo</span>
<a id="__codelineno-8-2" name="__codelineno-8-2" href="#__codelineno-8-2"></a>ssh-keygen<span class="w"> </span>-t<span class="w"> </span>ed25519<span class="w"> </span>-C<span class="w"> </span><span class="s2">&quot;argocd-deploy-key-helm-values&quot;</span><span class="w"> </span>-f<span class="w"> </span>argocd-helm-values-key<span class="w"> </span>-N<span class="w"> </span><span class="s2">&quot;&quot;</span>
<a id="__codelineno-8-3" name="__codelineno-8-3" href="#__codelineno-8-3"></a>
<a id="__codelineno-8-4" name="__codelineno-8-4" href="#__codelineno-8-4"></a><span class="c1"># Add to: https://github.com/fortedigital/helm-values/settings/keys</span>
</code></pre></div></p>
</li>
</ol>
<p><strong>Step 3: Create Kubernetes Secret</strong></p>
<p>Add the private key to ArgoCD as a repository secret:</p>
<p>Save the following file in private/ (gitignored) folder as secret.yaml
<div class="highlight"><pre><span></span><code><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="w"> </span>apiVersion:<span class="w"> </span>v1
<a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a><span class="w"> </span>kind:<span class="w"> </span>Secret
<a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a><span class="w"> </span>metadata:
<a id="__codelineno-9-4" name="__codelineno-9-4" href="#__codelineno-9-4"></a><span class="w"> </span>name:<span class="w"> </span>forte-helm-repo
<a id="__codelineno-9-5" name="__codelineno-9-5" href="#__codelineno-9-5"></a><span class="w"> </span>namespace:<span class="w"> </span>argocd
<a id="__codelineno-9-6" name="__codelineno-9-6" href="#__codelineno-9-6"></a><span class="w"> </span>labels:
<a id="__codelineno-9-7" name="__codelineno-9-7" href="#__codelineno-9-7"></a><span class="w"> </span>argocd.argoproj.io/secret-type:<span class="w"> </span>repository
<a id="__codelineno-9-8" name="__codelineno-9-8" href="#__codelineno-9-8"></a><span class="w"> </span>stringData:
<a id="__codelineno-9-9" name="__codelineno-9-9" href="#__codelineno-9-9"></a><span class="w"> </span>type:<span class="w"> </span>git
<a id="__codelineno-9-10" name="__codelineno-9-10" href="#__codelineno-9-10"></a><span class="w"> </span>url:<span class="w"> </span>ssh://git@git.forteapps.net:2222/Forte/forte-helm.git
<a id="__codelineno-9-11" name="__codelineno-9-11" href="#__codelineno-9-11"></a><span class="w"> </span>sshPrivateKey:<span class="w"> </span><span class="p">|</span>
<a id="__codelineno-9-12" name="__codelineno-9-12" href="#__codelineno-9-12"></a><span class="w"> </span>&lt;paste<span class="w"> </span>your<span class="w"> </span>private<span class="w"> </span>key<span class="w"> </span>here&gt;
<a id="__codelineno-9-13" name="__codelineno-9-13" href="#__codelineno-9-13"></a><span class="w"> </span>project:<span class="w"> </span>default
</code></pre></div>
Seal the secret using <code>kubeseal</code> command
<div class="highlight"><pre><span></span><code><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a>kubeseal<span class="w"> </span>--format<span class="o">=</span>yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a><span class="w"> </span>--namespace<span class="o">=</span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-10-3" name="__codelineno-10-3" href="#__codelineno-10-3"></a><span class="w"> </span>&lt;<span class="w"> </span>private/secret.yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-10-4" name="__codelineno-10-4" href="#__codelineno-10-4"></a><span class="w"> </span>&gt;<span class="w"> </span>secrets/forte-helm-repo-secret-sealed.yaml
</code></pre></div></p>
<p><strong>Step 4: Register Repository in ArgoCD</strong></p>
<p>Check in secrets/forte-helm-repo-secret-sealed.yaml and let Argo sync and create the secret.</p>
<p><strong>Step 5: Verify Repository Access</strong></p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a><span class="c1"># Check if repository is connected</span>
<a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secrets<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-l<span class="w"> </span>argocd.argoproj.io/secret-type<span class="o">=</span>repository
<a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a>
<a id="__codelineno-11-4" name="__codelineno-11-4" href="#__codelineno-11-4"></a><span class="c1"># Verify connection in ArgoCD UI</span>
<a id="__codelineno-11-5" name="__codelineno-11-5" href="#__codelineno-11-5"></a><span class="c1"># Settings → Repositories → Should show &quot;Successful&quot; status</span>
<a id="__codelineno-11-6" name="__codelineno-11-6" href="#__codelineno-11-6"></a>
<a id="__codelineno-11-7" name="__codelineno-11-7" href="#__codelineno-11-7"></a><span class="c1"># Test by creating an application</span>
<a id="__codelineno-11-8" name="__codelineno-11-8" href="#__codelineno-11-8"></a>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>_app-of-apps-upc-dev.yaml<span class="w"> </span><span class="c1"># or _app-of-apps-upc-prod.yaml</span>
<a id="__codelineno-11-9" name="__codelineno-11-9" href="#__codelineno-11-9"></a>
<a id="__codelineno-11-10" name="__codelineno-11-10" href="#__codelineno-11-10"></a><span class="c1"># Check application sync status</span>
<a id="__codelineno-11-11" name="__codelineno-11-11" href="#__codelineno-11-11"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd
</code></pre></div>
<h4 id="testing-repository-access">Testing Repository Access<a class="headerlink" href="#testing-repository-access" title="Permanent link">&para;</a></h4>
<p>Create a test application to verify SSH access:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a>cat<span class="w"> </span>&gt;<span class="w"> </span>/tmp/test-repo-access.yaml<span class="w"> </span><span class="s">&lt;&lt;EOF</span>
<a id="__codelineno-12-2" name="__codelineno-12-2" href="#__codelineno-12-2"></a><span class="s">apiVersion: argoproj.io/v1alpha1</span>
<a id="__codelineno-12-3" name="__codelineno-12-3" href="#__codelineno-12-3"></a><span class="s">kind: Application</span>
<a id="__codelineno-12-4" name="__codelineno-12-4" href="#__codelineno-12-4"></a><span class="s">metadata:</span>
<a id="__codelineno-12-5" name="__codelineno-12-5" href="#__codelineno-12-5"></a><span class="s"> name: test-repo-access</span>
<a id="__codelineno-12-6" name="__codelineno-12-6" href="#__codelineno-12-6"></a><span class="s"> namespace: argocd</span>
<a id="__codelineno-12-7" name="__codelineno-12-7" href="#__codelineno-12-7"></a><span class="s">spec:</span>
<a id="__codelineno-12-8" name="__codelineno-12-8" href="#__codelineno-12-8"></a><span class="s"> project: default</span>
<a id="__codelineno-12-9" name="__codelineno-12-9" href="#__codelineno-12-9"></a><span class="s"> source:</span>
<a id="__codelineno-12-10" name="__codelineno-12-10" href="#__codelineno-12-10"></a><span class="s"> repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git</span>
<a id="__codelineno-12-11" name="__codelineno-12-11" href="#__codelineno-12-11"></a><span class="s"> targetRevision: main</span>
<a id="__codelineno-12-12" name="__codelineno-12-12" href="#__codelineno-12-12"></a><span class="s"> path: cluster-resources</span>
<a id="__codelineno-12-13" name="__codelineno-12-13" href="#__codelineno-12-13"></a><span class="s"> destination:</span>
<a id="__codelineno-12-14" name="__codelineno-12-14" href="#__codelineno-12-14"></a><span class="s"> server: https://kubernetes.default.svc</span>
<a id="__codelineno-12-15" name="__codelineno-12-15" href="#__codelineno-12-15"></a><span class="s"> namespace: default</span>
<a id="__codelineno-12-16" name="__codelineno-12-16" href="#__codelineno-12-16"></a><span class="s"> syncPolicy:</span>
<a id="__codelineno-12-17" name="__codelineno-12-17" href="#__codelineno-12-17"></a><span class="s"> automated: null # Manual sync for testing</span>
<a id="__codelineno-12-18" name="__codelineno-12-18" href="#__codelineno-12-18"></a><span class="s">EOF</span>
<a id="__codelineno-12-19" name="__codelineno-12-19" href="#__codelineno-12-19"></a>
<a id="__codelineno-12-20" name="__codelineno-12-20" href="#__codelineno-12-20"></a>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>/tmp/test-repo-access.yaml
<a id="__codelineno-12-21" name="__codelineno-12-21" href="#__codelineno-12-21"></a>
<a id="__codelineno-12-22" name="__codelineno-12-22" href="#__codelineno-12-22"></a><span class="c1"># Check if ArgoCD can access the repository</span>
<a id="__codelineno-12-23" name="__codelineno-12-23" href="#__codelineno-12-23"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>application<span class="w"> </span>test-repo-access<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-12-24" name="__codelineno-12-24" href="#__codelineno-12-24"></a>
<a id="__codelineno-12-25" name="__codelineno-12-25" href="#__codelineno-12-25"></a><span class="c1"># Look for sync status - should show repository contents</span>
<a id="__codelineno-12-26" name="__codelineno-12-26" href="#__codelineno-12-26"></a>kubectl<span class="w"> </span>get<span class="w"> </span>application<span class="w"> </span>test-repo-access<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.status.sync.status}&#39;</span>
<a id="__codelineno-12-27" name="__codelineno-12-27" href="#__codelineno-12-27"></a>
<a id="__codelineno-12-28" name="__codelineno-12-28" href="#__codelineno-12-28"></a><span class="c1"># Clean up test application</span>
<a id="__codelineno-12-29" name="__codelineno-12-29" href="#__codelineno-12-29"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>application<span class="w"> </span>test-repo-access<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-12-30" name="__codelineno-12-30" href="#__codelineno-12-30"></a>rm<span class="w"> </span>/tmp/test-repo-access.yaml
</code></pre></div>
<h4 id="security-best-practices">Security Best Practices<a class="headerlink" href="#security-best-practices" title="Permanent link">&para;</a></h4>
<ol>
<li>
<p><strong>Secure Private Keys</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a><span class="c1"># Store private key securely and delete local copy</span>
<a id="__codelineno-13-2" name="__codelineno-13-2" href="#__codelineno-13-2"></a><span class="c1"># Option 1: Store in password manager (recommended)</span>
<a id="__codelineno-13-3" name="__codelineno-13-3" href="#__codelineno-13-3"></a><span class="c1"># Option 2: Backup to encrypted storage</span>
<a id="__codelineno-13-4" name="__codelineno-13-4" href="#__codelineno-13-4"></a>
<a id="__codelineno-13-5" name="__codelineno-13-5" href="#__codelineno-13-5"></a><span class="c1"># Delete local private key after adding to Kubernetes</span>
<a id="__codelineno-13-6" name="__codelineno-13-6" href="#__codelineno-13-6"></a>shred<span class="w"> </span>-u<span class="w"> </span>argocd-deploy-key
<a id="__codelineno-13-7" name="__codelineno-13-7" href="#__codelineno-13-7"></a>
<a id="__codelineno-13-8" name="__codelineno-13-8" href="#__codelineno-13-8"></a><span class="c1"># Or on Windows</span>
<a id="__codelineno-13-9" name="__codelineno-13-9" href="#__codelineno-13-9"></a><span class="c1"># Remove-Item -Path argocd-deploy-key -Force</span>
</code></pre></div></p>
</li>
<li>
<p><strong>Rotate Keys Regularly</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-14-1" name="__codelineno-14-1" href="#__codelineno-14-1"></a><span class="c1"># Generate new key</span>
<a id="__codelineno-14-2" name="__codelineno-14-2" href="#__codelineno-14-2"></a>ssh-keygen<span class="w"> </span>-t<span class="w"> </span>ed25519<span class="w"> </span>-C<span class="w"> </span><span class="s2">&quot;argocd-deploy-key-</span><span class="k">$(</span>date<span class="w"> </span>+%Y%m<span class="k">)</span><span class="s2">&quot;</span><span class="w"> </span>-f<span class="w"> </span>argocd-new-key<span class="w"> </span>-N<span class="w"> </span><span class="s2">&quot;&quot;</span>
<a id="__codelineno-14-3" name="__codelineno-14-3" href="#__codelineno-14-3"></a>
<a id="__codelineno-14-4" name="__codelineno-14-4" href="#__codelineno-14-4"></a><span class="c1"># Add new public key to GitHub (keep old key for now)</span>
<a id="__codelineno-14-5" name="__codelineno-14-5" href="#__codelineno-14-5"></a>
<a id="__codelineno-14-6" name="__codelineno-14-6" href="#__codelineno-14-6"></a><span class="c1"># Update Kubernetes secret</span>
<a id="__codelineno-14-7" name="__codelineno-14-7" href="#__codelineno-14-7"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>repo-launchpad<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-14-8" name="__codelineno-14-8" href="#__codelineno-14-8"></a><span class="w"> </span>--from-file<span class="o">=</span><span class="nv">sshPrivateKey</span><span class="o">=</span>argocd-new-key<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-14-9" name="__codelineno-14-9" href="#__codelineno-14-9"></a><span class="w"> </span>--namespace<span class="o">=</span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-14-10" name="__codelineno-14-10" href="#__codelineno-14-10"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>-
<a id="__codelineno-14-11" name="__codelineno-14-11" href="#__codelineno-14-11"></a>
<a id="__codelineno-14-12" name="__codelineno-14-12" href="#__codelineno-14-12"></a><span class="c1"># Test access, then remove old deploy key from GitHub</span>
<a id="__codelineno-14-13" name="__codelineno-14-13" href="#__codelineno-14-13"></a>
<a id="__codelineno-14-14" name="__codelineno-14-14" href="#__codelineno-14-14"></a><span class="c1"># Clean up</span>
<a id="__codelineno-14-15" name="__codelineno-14-15" href="#__codelineno-14-15"></a>shred<span class="w"> </span>-u<span class="w"> </span>argocd-new-key
</code></pre></div></p>
</li>
<li>
<p><strong>Audit Repository Access</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-15-1" name="__codelineno-15-1" href="#__codelineno-15-1"></a><span class="c1"># List all repository secrets</span>
<a id="__codelineno-15-2" name="__codelineno-15-2" href="#__codelineno-15-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secrets<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-l<span class="w"> </span>argocd.argoproj.io/secret-type<span class="o">=</span>repository
<a id="__codelineno-15-3" name="__codelineno-15-3" href="#__codelineno-15-3"></a>
<a id="__codelineno-15-4" name="__codelineno-15-4" href="#__codelineno-15-4"></a><span class="c1"># Review deploy keys in GitHub</span>
<a id="__codelineno-15-5" name="__codelineno-15-5" href="#__codelineno-15-5"></a><span class="c1"># Visit: https://git.forteapps.net/Forte/launchpad/settings/keys</span>
</code></pre></div></p>
</li>
<li>
<p><strong>Use Different Keys per Repository</strong></p>
</li>
<li>Don't reuse the same deploy key across repositories</li>
<li>If one key is compromised, only one repository is affected</li>
<li>Easier to track and audit access</li>
</ol>
<h4 id="troubleshooting-repository-access">Troubleshooting Repository Access<a class="headerlink" href="#troubleshooting-repository-access" title="Permanent link">&para;</a></h4>
<p><strong>Issue: "permission denied (publickey)"</strong></p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-16-1" name="__codelineno-16-1" href="#__codelineno-16-1"></a><span class="c1"># Check if secret exists</span>
<a id="__codelineno-16-2" name="__codelineno-16-2" href="#__codelineno-16-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>repo-launchpad<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-16-3" name="__codelineno-16-3" href="#__codelineno-16-3"></a>
<a id="__codelineno-16-4" name="__codelineno-16-4" href="#__codelineno-16-4"></a><span class="c1"># Verify secret has correct label</span>
<a id="__codelineno-16-5" name="__codelineno-16-5" href="#__codelineno-16-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>repo-launchpad<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>argocd.argoproj.io/secret-type
<a id="__codelineno-16-6" name="__codelineno-16-6" href="#__codelineno-16-6"></a>
<a id="__codelineno-16-7" name="__codelineno-16-7" href="#__codelineno-16-7"></a><span class="c1"># Check ArgoCD application controller logs</span>
<a id="__codelineno-16-8" name="__codelineno-16-8" href="#__codelineno-16-8"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>deployment/argocd-application-controller<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-i<span class="w"> </span><span class="s2">&quot;permission denied&quot;</span>
<a id="__codelineno-16-9" name="__codelineno-16-9" href="#__codelineno-16-9"></a>
<a id="__codelineno-16-10" name="__codelineno-16-10" href="#__codelineno-16-10"></a><span class="c1"># Verify deploy key is added to GitHub</span>
<a id="__codelineno-16-11" name="__codelineno-16-11" href="#__codelineno-16-11"></a><span class="c1"># Visit: https://git.forteapps.net/Forte/launchpad/settings/keys</span>
</code></pre></div>
<p><strong>Issue: "Host key verification failed"</strong></p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-17-1" name="__codelineno-17-1" href="#__codelineno-17-1"></a><span class="c1"># Add GitHub to known_hosts</span>
<a id="__codelineno-17-2" name="__codelineno-17-2" href="#__codelineno-17-2"></a>kubectl<span class="w"> </span><span class="nb">exec</span><span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>deployment/argocd-repo-server<span class="w"> </span>--<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-17-3" name="__codelineno-17-3" href="#__codelineno-17-3"></a><span class="w"> </span>ssh-keyscan<span class="w"> </span>github.com<span class="w"> </span>&gt;&gt;<span class="w"> </span>~/.ssh/known_hosts
<a id="__codelineno-17-4" name="__codelineno-17-4" href="#__codelineno-17-4"></a>
<a id="__codelineno-17-5" name="__codelineno-17-5" href="#__codelineno-17-5"></a><span class="c1"># Or disable strict host key checking (less secure)</span>
<a id="__codelineno-17-6" name="__codelineno-17-6" href="#__codelineno-17-6"></a>kubectl<span class="w"> </span>patch<span class="w"> </span>secret<span class="w"> </span>repo-launchpad<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-17-7" name="__codelineno-17-7" href="#__codelineno-17-7"></a><span class="w"> </span>--type<span class="w"> </span>merge<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-17-8" name="__codelineno-17-8" href="#__codelineno-17-8"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;stringData&quot;:{&quot;insecure&quot;:&quot;true&quot;}}&#39;</span>
</code></pre></div>
<p><strong>Issue: Repository shows as "Unknown" status</strong></p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-18-1" name="__codelineno-18-1" href="#__codelineno-18-1"></a><span class="c1"># Check repository server logs</span>
<a id="__codelineno-18-2" name="__codelineno-18-2" href="#__codelineno-18-2"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>deployment/argocd-repo-server
<a id="__codelineno-18-3" name="__codelineno-18-3" href="#__codelineno-18-3"></a>
<a id="__codelineno-18-4" name="__codelineno-18-4" href="#__codelineno-18-4"></a><span class="c1"># Refresh repository connection</span>
<a id="__codelineno-18-5" name="__codelineno-18-5" href="#__codelineno-18-5"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>secret<span class="w"> </span>repo-launchpad<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-18-6" name="__codelineno-18-6" href="#__codelineno-18-6"></a><span class="c1"># Recreate secret (see Step 3 above)</span>
<a id="__codelineno-18-7" name="__codelineno-18-7" href="#__codelineno-18-7"></a>
<a id="__codelineno-18-8" name="__codelineno-18-8" href="#__codelineno-18-8"></a><span class="c1"># Restart ArgoCD components</span>
<a id="__codelineno-18-9" name="__codelineno-18-9" href="#__codelineno-18-9"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>restart<span class="w"> </span>deployment<span class="w"> </span>argocd-repo-server<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-18-10" name="__codelineno-18-10" href="#__codelineno-18-10"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>restart<span class="w"> </span>deployment<span class="w"> </span>argocd-application-controller<span class="w"> </span>-n<span class="w"> </span>argocd
</code></pre></div>
<h4 id="multiple-repository-setup">Multiple Repository Setup<a class="headerlink" href="#multiple-repository-setup" title="Permanent link">&para;</a></h4>
<p>For the three-repository pattern (launchpad, forte-helm, helm-values):</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-19-1" name="__codelineno-19-1" href="#__codelineno-19-1"></a><span class="c1"># 1. launchpad (main config repo)</span>
<a id="__codelineno-19-2" name="__codelineno-19-2" href="#__codelineno-19-2"></a>ssh-keygen<span class="w"> </span>-t<span class="w"> </span>ed25519<span class="w"> </span>-C<span class="w"> </span><span class="s2">&quot;argocd-launchpad&quot;</span><span class="w"> </span>-f<span class="w"> </span>key-sturdy<span class="w"> </span>-N<span class="w"> </span><span class="s2">&quot;&quot;</span>
<a id="__codelineno-19-3" name="__codelineno-19-3" href="#__codelineno-19-3"></a><span class="c1"># Add key-sturdy.pub to: https://git.forteapps.net/Forte/launchpad/settings/keys</span>
<a id="__codelineno-19-4" name="__codelineno-19-4" href="#__codelineno-19-4"></a>
<a id="__codelineno-19-5" name="__codelineno-19-5" href="#__codelineno-19-5"></a><span class="c1"># 2. helm-values (private values repo)</span>
<a id="__codelineno-19-6" name="__codelineno-19-6" href="#__codelineno-19-6"></a>ssh-keygen<span class="w"> </span>-t<span class="w"> </span>ed25519<span class="w"> </span>-C<span class="w"> </span><span class="s2">&quot;argocd-helm-values&quot;</span><span class="w"> </span>-f<span class="w"> </span>key-helm-values<span class="w"> </span>-N<span class="w"> </span><span class="s2">&quot;&quot;</span>
<a id="__codelineno-19-7" name="__codelineno-19-7" href="#__codelineno-19-7"></a><span class="c1"># Add key-helm-values.pub to: https://github.com/fortedigital/helm-values/settings/keys</span>
<a id="__codelineno-19-8" name="__codelineno-19-8" href="#__codelineno-19-8"></a>
<a id="__codelineno-19-9" name="__codelineno-19-9" href="#__codelineno-19-9"></a><span class="c1"># 3. forte-helm (private helm charts repo)</span>
<a id="__codelineno-19-10" name="__codelineno-19-10" href="#__codelineno-19-10"></a>
<a id="__codelineno-19-11" name="__codelineno-19-11" href="#__codelineno-19-11"></a><span class="c1"># Create secrets</span>
<a id="__codelineno-19-12" name="__codelineno-19-12" href="#__codelineno-19-12"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>repo-launchpad<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-13" name="__codelineno-19-13" href="#__codelineno-19-13"></a><span class="w"> </span>--from-file<span class="o">=</span><span class="nv">sshPrivateKey</span><span class="o">=</span>key-sturdy<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-14" name="__codelineno-19-14" href="#__codelineno-19-14"></a><span class="w"> </span>--namespace<span class="o">=</span>argocd<span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-15" name="__codelineno-19-15" href="#__codelineno-19-15"></a><span class="w"> </span>kubectl<span class="w"> </span>label<span class="w"> </span>--local<span class="w"> </span>-f<span class="w"> </span>-<span class="w"> </span>argocd.argoproj.io/secret-type<span class="o">=</span>repository<span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-16" name="__codelineno-19-16" href="#__codelineno-19-16"></a><span class="w"> </span>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>-
<a id="__codelineno-19-17" name="__codelineno-19-17" href="#__codelineno-19-17"></a>
<a id="__codelineno-19-18" name="__codelineno-19-18" href="#__codelineno-19-18"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>repo-helm-values<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-19" name="__codelineno-19-19" href="#__codelineno-19-19"></a><span class="w"> </span>--from-file<span class="o">=</span><span class="nv">sshPrivateKey</span><span class="o">=</span>key-helm-values<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-20" name="__codelineno-19-20" href="#__codelineno-19-20"></a><span class="w"> </span>--namespace<span class="o">=</span>argocd<span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-21" name="__codelineno-19-21" href="#__codelineno-19-21"></a><span class="w"> </span>kubectl<span class="w"> </span>label<span class="w"> </span>--local<span class="w"> </span>-f<span class="w"> </span>-<span class="w"> </span>argocd.argoproj.io/secret-type<span class="o">=</span>repository<span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-19-22" name="__codelineno-19-22" href="#__codelineno-19-22"></a><span class="w"> </span>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>-
<a id="__codelineno-19-23" name="__codelineno-19-23" href="#__codelineno-19-23"></a>
<a id="__codelineno-19-24" name="__codelineno-19-24" href="#__codelineno-19-24"></a><span class="c1"># Clean up keys</span>
<a id="__codelineno-19-25" name="__codelineno-19-25" href="#__codelineno-19-25"></a>shred<span class="w"> </span>-u<span class="w"> </span>key-sturdy<span class="w"> </span>key-helm-values
</code></pre></div>
<h4 id="converting-https-to-ssh">Converting HTTPS to SSH<a class="headerlink" href="#converting-https-to-ssh" title="Permanent link">&para;</a></h4>
<p>If you're currently using HTTPS and want to switch to SSH:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-20-1" name="__codelineno-20-1" href="#__codelineno-20-1"></a><span class="c1"># 1. Generate and add deploy key (see steps above)</span>
<a id="__codelineno-20-2" name="__codelineno-20-2" href="#__codelineno-20-2"></a>
<a id="__codelineno-20-3" name="__codelineno-20-3" href="#__codelineno-20-3"></a><span class="c1"># 2. Update all Application manifests</span>
<a id="__codelineno-20-4" name="__codelineno-20-4" href="#__codelineno-20-4"></a><span class="c1"># Change from:</span>
<a id="__codelineno-20-5" name="__codelineno-20-5" href="#__codelineno-20-5"></a><span class="c1"># repoURL: https://git.forteapps.net/Forte/launchpad</span>
<a id="__codelineno-20-6" name="__codelineno-20-6" href="#__codelineno-20-6"></a><span class="c1"># To:</span>
<a id="__codelineno-20-7" name="__codelineno-20-7" href="#__codelineno-20-7"></a><span class="c1"># repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git</span>
<a id="__codelineno-20-8" name="__codelineno-20-8" href="#__codelineno-20-8"></a>
<a id="__codelineno-20-9" name="__codelineno-20-9" href="#__codelineno-20-9"></a><span class="c1"># 3. Update and commit</span>
<a id="__codelineno-20-10" name="__codelineno-20-10" href="#__codelineno-20-10"></a>find<span class="w"> </span>.<span class="w"> </span>-name<span class="w"> </span><span class="s2">&quot;*.yaml&quot;</span><span class="w"> </span>-type<span class="w"> </span>f<span class="w"> </span>-exec<span class="w"> </span>sed<span class="w"> </span>-i<span class="w"> </span><span class="s1">&#39;s|https://github.com/fortedigital/|git@github.com:fortedigital/|g&#39;</span><span class="w"> </span><span class="o">{}</span><span class="w"> </span>+
<a id="__codelineno-20-11" name="__codelineno-20-11" href="#__codelineno-20-11"></a>
<a id="__codelineno-20-12" name="__codelineno-20-12" href="#__codelineno-20-12"></a>git<span class="w"> </span>add<span class="w"> </span>.
<a id="__codelineno-20-13" name="__codelineno-20-13" href="#__codelineno-20-13"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Switch from HTTPS to SSH for repository access&quot;</span>
<a id="__codelineno-20-14" name="__codelineno-20-14" href="#__codelineno-20-14"></a>git<span class="w"> </span>push
<a id="__codelineno-20-15" name="__codelineno-20-15" href="#__codelineno-20-15"></a>
<a id="__codelineno-20-16" name="__codelineno-20-16" href="#__codelineno-20-16"></a><span class="c1"># 4. ArgoCD will automatically re-sync with new SSH URLs</span>
</code></pre></div>
<hr />
<h2 id="day-to-day-operations">Day-to-Day Operations<a class="headerlink" href="#day-to-day-operations" title="Permanent link">&para;</a></h2>
<h3 id="monitoring-argocd-sync-status">Monitoring ArgoCD Sync Status<a class="headerlink" href="#monitoring-argocd-sync-status" title="Permanent link">&para;</a></h3>
<h4 id="via-slack">Via Slack<a class="headerlink" href="#via-slack" title="Permanent link">&para;</a></h4>
<p>All applications send notifications to shared Slack channel:
- ✅ <code>on-sync-succeeded</code> - Deployment succeeded
- ❌ <code>on-sync-failed</code> - Deployment failed
- ⚠️ <code>on-degraded</code> - Application unhealthy</p>
<h4 id="via-cli">Via CLI<a class="headerlink" href="#via-cli" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-21-1" name="__codelineno-21-1" href="#__codelineno-21-1"></a><span class="c1"># List all applications</span>
<a id="__codelineno-21-2" name="__codelineno-21-2" href="#__codelineno-21-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-21-3" name="__codelineno-21-3" href="#__codelineno-21-3"></a>
<a id="__codelineno-21-4" name="__codelineno-21-4" href="#__codelineno-21-4"></a><span class="c1"># Watch application status</span>
<a id="__codelineno-21-5" name="__codelineno-21-5" href="#__codelineno-21-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-w
<a id="__codelineno-21-6" name="__codelineno-21-6" href="#__codelineno-21-6"></a>
<a id="__codelineno-21-7" name="__codelineno-21-7" href="#__codelineno-21-7"></a><span class="c1"># Get detailed status</span>
<a id="__codelineno-21-8" name="__codelineno-21-8" href="#__codelineno-21-8"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd
</code></pre></div>
<h4 id="via-argocd-ui">Via ArgoCD UI<a class="headerlink" href="#via-argocd-ui" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-22-1" name="__codelineno-22-1" href="#__codelineno-22-1"></a><span class="c1"># Port forward to UI</span>
<a id="__codelineno-22-2" name="__codelineno-22-2" href="#__codelineno-22-2"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>svc/argocd-server<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="m">8080</span>:443
<a id="__codelineno-22-3" name="__codelineno-22-3" href="#__codelineno-22-3"></a>
<a id="__codelineno-22-4" name="__codelineno-22-4" href="#__codelineno-22-4"></a><span class="c1"># Access: https://localhost:8080</span>
<a id="__codelineno-22-5" name="__codelineno-22-5" href="#__codelineno-22-5"></a><span class="c1"># No login required (insecure mode for internal use)</span>
</code></pre></div>
<h3 id="checking-application-health">Checking Application Health<a class="headerlink" href="#checking-application-health" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-23-1" name="__codelineno-23-1" href="#__codelineno-23-1"></a><span class="c1"># Quick health check for all apps</span>
<a id="__codelineno-23-2" name="__codelineno-23-2" href="#__codelineno-23-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-23-3" name="__codelineno-23-3" href="#__codelineno-23-3"></a><span class="w"> </span>-o<span class="w"> </span>custom-columns<span class="o">=</span>NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status
<a id="__codelineno-23-4" name="__codelineno-23-4" href="#__codelineno-23-4"></a>
<a id="__codelineno-23-5" name="__codelineno-23-5" href="#__codelineno-23-5"></a><span class="c1"># Expected output:</span>
<a id="__codelineno-23-6" name="__codelineno-23-6" href="#__codelineno-23-6"></a><span class="c1"># NAME SYNC HEALTH</span>
<a id="__codelineno-23-7" name="__codelineno-23-7" href="#__codelineno-23-7"></a><span class="c1"># infrastructure-apps Synced Healthy</span>
<a id="__codelineno-23-8" name="__codelineno-23-8" href="#__codelineno-23-8"></a><span class="c1"># enterprise-apps Synced Healthy</span>
<a id="__codelineno-23-9" name="__codelineno-23-9" href="#__codelineno-23-9"></a><span class="c1"># mcp10x Synced Healthy</span>
<a id="__codelineno-23-10" name="__codelineno-23-10" href="#__codelineno-23-10"></a><span class="c1"># musicman Synced Healthy</span>
</code></pre></div>
<h3 id="manual-sync">Manual Sync<a class="headerlink" href="#manual-sync" title="Permanent link">&para;</a></h3>
<p>Force sync an application:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-24-1" name="__codelineno-24-1" href="#__codelineno-24-1"></a><span class="c1"># Trigger sync</span>
<a id="__codelineno-24-2" name="__codelineno-24-2" href="#__codelineno-24-2"></a>kubectl<span class="w"> </span>patch<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-24-3" name="__codelineno-24-3" href="#__codelineno-24-3"></a><span class="w"> </span>--type<span class="w"> </span>merge<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-24-4" name="__codelineno-24-4" href="#__codelineno-24-4"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;metadata&quot;:{&quot;annotations&quot;:{&quot;argocd.argoproj.io/refresh&quot;:&quot;hard&quot;}}}&#39;</span>
<a id="__codelineno-24-5" name="__codelineno-24-5" href="#__codelineno-24-5"></a>
<a id="__codelineno-24-6" name="__codelineno-24-6" href="#__codelineno-24-6"></a><span class="c1"># Or via ArgoCD CLI (if installed)</span>
<a id="__codelineno-24-7" name="__codelineno-24-7" href="#__codelineno-24-7"></a>argocd<span class="w"> </span>app<span class="w"> </span>sync<span class="w"> </span>myapp
</code></pre></div>
<h3 id="pausing-auto-sync">Pausing Auto-Sync<a class="headerlink" href="#pausing-auto-sync" title="Permanent link">&para;</a></h3>
<p>Temporarily disable automatic syncing:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-25-1" name="__codelineno-25-1" href="#__codelineno-25-1"></a><span class="c1"># Edit application</span>
<a id="__codelineno-25-2" name="__codelineno-25-2" href="#__codelineno-25-2"></a>kubectl<span class="w"> </span>edit<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-25-3" name="__codelineno-25-3" href="#__codelineno-25-3"></a>
<a id="__codelineno-25-4" name="__codelineno-25-4" href="#__codelineno-25-4"></a><span class="c1"># Set automated to null</span>
<a id="__codelineno-25-5" name="__codelineno-25-5" href="#__codelineno-25-5"></a>spec:
<a id="__codelineno-25-6" name="__codelineno-25-6" href="#__codelineno-25-6"></a><span class="w"> </span>syncPolicy:
<a id="__codelineno-25-7" name="__codelineno-25-7" href="#__codelineno-25-7"></a><span class="w"> </span>automated:<span class="w"> </span>null<span class="w"> </span><span class="c1"># Disable auto-sync</span>
<a id="__codelineno-25-8" name="__codelineno-25-8" href="#__codelineno-25-8"></a>
<a id="__codelineno-25-9" name="__codelineno-25-9" href="#__codelineno-25-9"></a><span class="c1"># Re-enable later</span>
<a id="__codelineno-25-10" name="__codelineno-25-10" href="#__codelineno-25-10"></a>spec:
<a id="__codelineno-25-11" name="__codelineno-25-11" href="#__codelineno-25-11"></a><span class="w"> </span>syncPolicy:
<a id="__codelineno-25-12" name="__codelineno-25-12" href="#__codelineno-25-12"></a><span class="w"> </span>automated:
<a id="__codelineno-25-13" name="__codelineno-25-13" href="#__codelineno-25-13"></a><span class="w"> </span>prune:<span class="w"> </span><span class="nb">true</span>
<a id="__codelineno-25-14" name="__codelineno-25-14" href="#__codelineno-25-14"></a><span class="w"> </span>selfHeal:<span class="w"> </span><span class="nb">true</span>
</code></pre></div>
<hr />
<h2 id="application-management">Application Management<a class="headerlink" href="#application-management" title="Permanent link">&para;</a></h2>
<h3 id="deploying-a-new-application">Deploying a New Application<a class="headerlink" href="#deploying-a-new-application" title="Permanent link">&para;</a></h3>
<p>See <a href="../DEVELOPER-GUIDE/#deploying-your-first-application">Developer Guide</a> for detailed steps.</p>
<p><strong>Quick checklist:</strong>
- [ ] Create <code>helm-values/myapp/values.yaml</code>
- [ ] Create <code>apps/myapp.yaml</code> in config repo
- [ ] Create SealedSecret if needed
- [ ] Commit and push changes
- [ ] Verify sync in Slack/ArgoCD
- [ ] Configure DNS for domain
- [ ] Test application accessibility</p>
<h3 id="removing-an-application">Removing an Application<a class="headerlink" href="#removing-an-application" title="Permanent link">&para;</a></h3>
<h4 id="safe-removal-procedure">Safe Removal Procedure<a class="headerlink" href="#safe-removal-procedure" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-26-1" name="__codelineno-26-1" href="#__codelineno-26-1"></a><span class="c1"># 1. Delete ArgoCD Application (with cascade)</span>
<a id="__codelineno-26-2" name="__codelineno-26-2" href="#__codelineno-26-2"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-26-3" name="__codelineno-26-3" href="#__codelineno-26-3"></a>
<a id="__codelineno-26-4" name="__codelineno-26-4" href="#__codelineno-26-4"></a><span class="c1"># This will:</span>
<a id="__codelineno-26-5" name="__codelineno-26-5" href="#__codelineno-26-5"></a><span class="c1"># - Remove application from ArgoCD</span>
<a id="__codelineno-26-6" name="__codelineno-26-6" href="#__codelineno-26-6"></a><span class="c1"># - Delete all Kubernetes resources (cascade)</span>
<a id="__codelineno-26-7" name="__codelineno-26-7" href="#__codelineno-26-7"></a><span class="c1"># - Remove namespace</span>
<a id="__codelineno-26-8" name="__codelineno-26-8" href="#__codelineno-26-8"></a>
<a id="__codelineno-26-9" name="__codelineno-26-9" href="#__codelineno-26-9"></a><span class="c1"># 2. Clean up Git repositories</span>
<a id="__codelineno-26-10" name="__codelineno-26-10" href="#__codelineno-26-10"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/launchpad
<a id="__codelineno-26-11" name="__codelineno-26-11" href="#__codelineno-26-11"></a>git<span class="w"> </span>rm<span class="w"> </span>apps/myapp.yaml
<a id="__codelineno-26-12" name="__codelineno-26-12" href="#__codelineno-26-12"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Remove myapp application&quot;</span>
<a id="__codelineno-26-13" name="__codelineno-26-13" href="#__codelineno-26-13"></a>git<span class="w"> </span>push
<a id="__codelineno-26-14" name="__codelineno-26-14" href="#__codelineno-26-14"></a>
<a id="__codelineno-26-15" name="__codelineno-26-15" href="#__codelineno-26-15"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/helm-prod-values
<a id="__codelineno-26-16" name="__codelineno-26-16" href="#__codelineno-26-16"></a>git<span class="w"> </span>rm<span class="w"> </span>-r<span class="w"> </span>myapp/
<a id="__codelineno-26-17" name="__codelineno-26-17" href="#__codelineno-26-17"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Remove myapp values&quot;</span>
<a id="__codelineno-26-18" name="__codelineno-26-18" href="#__codelineno-26-18"></a>git<span class="w"> </span>push
<a id="__codelineno-26-19" name="__codelineno-26-19" href="#__codelineno-26-19"></a>
<a id="__codelineno-26-20" name="__codelineno-26-20" href="#__codelineno-26-20"></a><span class="c1"># 3. Remove sealed secrets (if any)</span>
<a id="__codelineno-26-21" name="__codelineno-26-21" href="#__codelineno-26-21"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/launchpad
<a id="__codelineno-26-22" name="__codelineno-26-22" href="#__codelineno-26-22"></a>git<span class="w"> </span>rm<span class="w"> </span>secrets/myapp-credentials-sealed.yaml
<a id="__codelineno-26-23" name="__codelineno-26-23" href="#__codelineno-26-23"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Remove myapp secrets&quot;</span>
<a id="__codelineno-26-24" name="__codelineno-26-24" href="#__codelineno-26-24"></a>git<span class="w"> </span>push
</code></pre></div>
<h4 id="removal-without-cascade">Removal Without Cascade<a class="headerlink" href="#removal-without-cascade" title="Permanent link">&para;</a></h4>
<p>To remove from ArgoCD but keep resources running:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-27-1" name="__codelineno-27-1" href="#__codelineno-27-1"></a><span class="c1"># Delete application with no cascade</span>
<a id="__codelineno-27-2" name="__codelineno-27-2" href="#__codelineno-27-2"></a>kubectl<span class="w"> </span>patch<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-27-3" name="__codelineno-27-3" href="#__codelineno-27-3"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;metadata&quot;:{&quot;finalizers&quot;:[]}}&#39;</span><span class="w"> </span>--type<span class="w"> </span>merge
<a id="__codelineno-27-4" name="__codelineno-27-4" href="#__codelineno-27-4"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-27-5" name="__codelineno-27-5" href="#__codelineno-27-5"></a>
<a id="__codelineno-27-6" name="__codelineno-27-6" href="#__codelineno-27-6"></a><span class="c1"># Resources remain in cluster but are no longer managed</span>
</code></pre></div>
<h3 id="scaling-applications">Scaling Applications<a class="headerlink" href="#scaling-applications" title="Permanent link">&para;</a></h3>
<h4 id="manual-scaling">Manual Scaling<a class="headerlink" href="#manual-scaling" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-28-1" name="__codelineno-28-1" href="#__codelineno-28-1"></a><span class="c1"># Scale deployment directly</span>
<a id="__codelineno-28-2" name="__codelineno-28-2" href="#__codelineno-28-2"></a>kubectl<span class="w"> </span>scale<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>--replicas<span class="o">=</span><span class="m">3</span>
<a id="__codelineno-28-3" name="__codelineno-28-3" href="#__codelineno-28-3"></a>
<a id="__codelineno-28-4" name="__codelineno-28-4" href="#__codelineno-28-4"></a><span class="c1"># Note: If selfHeal is enabled, this will be reverted</span>
</code></pre></div>
<h4 id="gitops-scaling">GitOps Scaling<a class="headerlink" href="#gitops-scaling" title="Permanent link">&para;</a></h4>
<p>Update <code>helm-values/myapp/values.yaml</code>:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-29-1" name="__codelineno-29-1" href="#__codelineno-29-1"></a><span class="nt">app</span><span class="p">:</span>
<a id="__codelineno-29-2" name="__codelineno-29-2" href="#__codelineno-29-2"></a><span class="w"> </span><span class="nt">replicaCount</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">3</span><span class="w"> </span><span class="c1"># Change from 1 to 3</span>
</code></pre></div>
<p>Commit and push - ArgoCD will sync.</p>
<h4 id="auto-scaling-hpa">Auto-Scaling (HPA)<a class="headerlink" href="#auto-scaling-hpa" title="Permanent link">&para;</a></h4>
<p>Enable Horizontal Pod Autoscaler:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-30-1" name="__codelineno-30-1" href="#__codelineno-30-1"></a><span class="c1"># In helm-values/myapp/values.yaml</span>
<a id="__codelineno-30-2" name="__codelineno-30-2" href="#__codelineno-30-2"></a><span class="nt">app</span><span class="p">:</span>
<a id="__codelineno-30-3" name="__codelineno-30-3" href="#__codelineno-30-3"></a><span class="w"> </span><span class="nt">hpa</span><span class="p">:</span>
<a id="__codelineno-30-4" name="__codelineno-30-4" href="#__codelineno-30-4"></a><span class="w"> </span><span class="nt">enabled</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
<a id="__codelineno-30-5" name="__codelineno-30-5" href="#__codelineno-30-5"></a><span class="w"> </span><span class="nt">minReplicas</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2</span>
<a id="__codelineno-30-6" name="__codelineno-30-6" href="#__codelineno-30-6"></a><span class="w"> </span><span class="nt">maxReplicas</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10</span>
<a id="__codelineno-30-7" name="__codelineno-30-7" href="#__codelineno-30-7"></a><span class="w"> </span><span class="nt">targetCPUUtilizationPercentage</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">70</span>
</code></pre></div>
<p><strong>Note:</strong> Remove <code>replicaCount</code> from ArgoCD ignore list if using HPA:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-31-1" name="__codelineno-31-1" href="#__codelineno-31-1"></a><span class="c1"># In apps/myapp.yaml</span>
<a id="__codelineno-31-2" name="__codelineno-31-2" href="#__codelineno-31-2"></a><span class="nt">ignoreDifferences</span><span class="p">:</span>
<a id="__codelineno-31-3" name="__codelineno-31-3" href="#__codelineno-31-3"></a><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">group</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">apps</span>
<a id="__codelineno-31-4" name="__codelineno-31-4" href="#__codelineno-31-4"></a><span class="w"> </span><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Deployment</span>
<a id="__codelineno-31-5" name="__codelineno-31-5" href="#__codelineno-31-5"></a><span class="w"> </span><span class="nt">jsonPointers</span><span class="p">:</span>
<a id="__codelineno-31-6" name="__codelineno-31-6" href="#__codelineno-31-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/spec/replicas</span><span class="w"> </span><span class="c1"># Remove this line</span>
</code></pre></div>
<h3 id="rolling-back-deployments">Rolling Back Deployments<a class="headerlink" href="#rolling-back-deployments" title="Permanent link">&para;</a></h3>
<h4 id="option-1-git-revert">Option 1: Git Revert<a class="headerlink" href="#option-1-git-revert" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-32-1" name="__codelineno-32-1" href="#__codelineno-32-1"></a><span class="c1"># Find the commit before the bad change</span>
<a id="__codelineno-32-2" name="__codelineno-32-2" href="#__codelineno-32-2"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/helm-prod-values
<a id="__codelineno-32-3" name="__codelineno-32-3" href="#__codelineno-32-3"></a>git<span class="w"> </span>log<span class="w"> </span>--oneline<span class="w"> </span>myapp/values.yaml
<a id="__codelineno-32-4" name="__codelineno-32-4" href="#__codelineno-32-4"></a>
<a id="__codelineno-32-5" name="__codelineno-32-5" href="#__codelineno-32-5"></a><span class="c1"># Revert to previous version</span>
<a id="__codelineno-32-6" name="__codelineno-32-6" href="#__codelineno-32-6"></a>git<span class="w"> </span>revert<span class="w"> </span>&lt;commit-hash&gt;
<a id="__codelineno-32-7" name="__codelineno-32-7" href="#__codelineno-32-7"></a>git<span class="w"> </span>push
<a id="__codelineno-32-8" name="__codelineno-32-8" href="#__codelineno-32-8"></a>
<a id="__codelineno-32-9" name="__codelineno-32-9" href="#__codelineno-32-9"></a><span class="c1"># ArgoCD will sync the rollback</span>
</code></pre></div>
<h4 id="option-2-manual-rollback">Option 2: Manual Rollback<a class="headerlink" href="#option-2-manual-rollback" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-33-1" name="__codelineno-33-1" href="#__codelineno-33-1"></a><span class="c1"># Rollback to previous revision</span>
<a id="__codelineno-33-2" name="__codelineno-33-2" href="#__codelineno-33-2"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>undo<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-33-3" name="__codelineno-33-3" href="#__codelineno-33-3"></a>
<a id="__codelineno-33-4" name="__codelineno-33-4" href="#__codelineno-33-4"></a><span class="c1"># Note: This will be reverted by ArgoCD selfHeal</span>
<a id="__codelineno-33-5" name="__codelineno-33-5" href="#__codelineno-33-5"></a><span class="c1"># Make permanent by updating Git</span>
</code></pre></div>
<h4 id="option-3-change-image-tag">Option 3: Change Image Tag<a class="headerlink" href="#option-3-change-image-tag" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-34-1" name="__codelineno-34-1" href="#__codelineno-34-1"></a><span class="c1"># Edit helm-values</span>
<a id="__codelineno-34-2" name="__codelineno-34-2" href="#__codelineno-34-2"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/helm-prod-values
<a id="__codelineno-34-3" name="__codelineno-34-3" href="#__codelineno-34-3"></a>vim<span class="w"> </span>myapp/values.yaml
<a id="__codelineno-34-4" name="__codelineno-34-4" href="#__codelineno-34-4"></a>
<a id="__codelineno-34-5" name="__codelineno-34-5" href="#__codelineno-34-5"></a><span class="c1"># Change image tag to previous version</span>
<a id="__codelineno-34-6" name="__codelineno-34-6" href="#__codelineno-34-6"></a>app:
<a id="__codelineno-34-7" name="__codelineno-34-7" href="#__codelineno-34-7"></a><span class="w"> </span>image:
<a id="__codelineno-34-8" name="__codelineno-34-8" href="#__codelineno-34-8"></a><span class="w"> </span>tag:<span class="w"> </span>v1.0.0<span class="w"> </span><span class="c1"># Roll back from v1.0.1</span>
<a id="__codelineno-34-9" name="__codelineno-34-9" href="#__codelineno-34-9"></a>
<a id="__codelineno-34-10" name="__codelineno-34-10" href="#__codelineno-34-10"></a><span class="c1"># Commit and push</span>
<a id="__codelineno-34-11" name="__codelineno-34-11" href="#__codelineno-34-11"></a>git<span class="w"> </span>add<span class="w"> </span>myapp/values.yaml
<a id="__codelineno-34-12" name="__codelineno-34-12" href="#__codelineno-34-12"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Rollback myapp to v1.0.0&quot;</span>
<a id="__codelineno-34-13" name="__codelineno-34-13" href="#__codelineno-34-13"></a>git<span class="w"> </span>push
</code></pre></div>
<h3 id="resource-updates">Resource Updates<a class="headerlink" href="#resource-updates" title="Permanent link">&para;</a></h3>
<h4 id="update-resource-limits">Update Resource Limits<a class="headerlink" href="#update-resource-limits" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-35-1" name="__codelineno-35-1" href="#__codelineno-35-1"></a><span class="c1"># In helm-values/myapp/values.yaml</span>
<a id="__codelineno-35-2" name="__codelineno-35-2" href="#__codelineno-35-2"></a><span class="nt">app</span><span class="p">:</span>
<a id="__codelineno-35-3" name="__codelineno-35-3" href="#__codelineno-35-3"></a><span class="w"> </span><span class="nt">resources</span><span class="p">:</span>
<a id="__codelineno-35-4" name="__codelineno-35-4" href="#__codelineno-35-4"></a><span class="w"> </span><span class="nt">requests</span><span class="p">:</span>
<a id="__codelineno-35-5" name="__codelineno-35-5" href="#__codelineno-35-5"></a><span class="w"> </span><span class="nt">cpu</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">200m</span><span class="w"> </span><span class="c1"># Increased from 100m</span>
<a id="__codelineno-35-6" name="__codelineno-35-6" href="#__codelineno-35-6"></a><span class="w"> </span><span class="nt">memory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">512Mi</span><span class="w"> </span><span class="c1"># Increased from 256Mi</span>
<a id="__codelineno-35-7" name="__codelineno-35-7" href="#__codelineno-35-7"></a><span class="w"> </span><span class="nt">limits</span><span class="p">:</span>
<a id="__codelineno-35-8" name="__codelineno-35-8" href="#__codelineno-35-8"></a><span class="w"> </span><span class="nt">cpu</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">1000m</span>
<a id="__codelineno-35-9" name="__codelineno-35-9" href="#__codelineno-35-9"></a><span class="w"> </span><span class="nt">memory</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">2Gi</span>
</code></pre></div>
<h4 id="enable-database">Enable Database<a class="headerlink" href="#enable-database" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-36-1" name="__codelineno-36-1" href="#__codelineno-36-1"></a><span class="c1"># In helm-values/myapp/values.yaml</span>
<a id="__codelineno-36-2" name="__codelineno-36-2" href="#__codelineno-36-2"></a><span class="nt">db</span><span class="p">:</span>
<a id="__codelineno-36-3" name="__codelineno-36-3" href="#__codelineno-36-3"></a><span class="w"> </span><span class="nt">enabled</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">true</span>
<a id="__codelineno-36-4" name="__codelineno-36-4" href="#__codelineno-36-4"></a><span class="w"> </span><span class="nt">persistence</span><span class="p">:</span>
<a id="__codelineno-36-5" name="__codelineno-36-5" href="#__codelineno-36-5"></a><span class="w"> </span><span class="nt">size</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">10Gi</span><span class="w"> </span><span class="c1"># Increase storage</span>
</code></pre></div>
<hr />
<h2 id="secret-management">Secret Management<a class="headerlink" href="#secret-management" title="Permanent link">&para;</a></h2>
<h3 id="creating-secrets">Creating Secrets<a class="headerlink" href="#creating-secrets" title="Permanent link">&para;</a></h3>
<h4 id="step-1-get-public-certificate">Step 1: Get Public Certificate<a class="headerlink" href="#step-1-get-public-certificate" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-37-1" name="__codelineno-37-1" href="#__codelineno-37-1"></a><span class="c1"># Fetch sealed-secrets public cert (one-time)</span>
<a id="__codelineno-37-2" name="__codelineno-37-2" href="#__codelineno-37-2"></a>kubeseal<span class="w"> </span>--fetch-cert<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-37-3" name="__codelineno-37-3" href="#__codelineno-37-3"></a><span class="w"> </span>--controller-name<span class="o">=</span>sealed-secrets-controller<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-37-4" name="__codelineno-37-4" href="#__codelineno-37-4"></a><span class="w"> </span>--controller-namespace<span class="o">=</span>kube-system<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-37-5" name="__codelineno-37-5" href="#__codelineno-37-5"></a><span class="w"> </span>&gt;<span class="w"> </span>pub-cert.pem
<a id="__codelineno-37-6" name="__codelineno-37-6" href="#__codelineno-37-6"></a>
<a id="__codelineno-37-7" name="__codelineno-37-7" href="#__codelineno-37-7"></a><span class="c1"># Save this certificate for future use</span>
</code></pre></div>
<h4 id="step-2-create-plain-secret">Step 2: Create Plain Secret<a class="headerlink" href="#step-2-create-plain-secret" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-38-1" name="__codelineno-38-1" href="#__codelineno-38-1"></a><span class="c1"># Method 1: From literal values</span>
<a id="__codelineno-38-2" name="__codelineno-38-2" href="#__codelineno-38-2"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>myapp-credentials<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-3" name="__codelineno-38-3" href="#__codelineno-38-3"></a><span class="w"> </span>--from-literal<span class="o">=</span><span class="nv">API_KEY</span><span class="o">=</span>secret123<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-4" name="__codelineno-38-4" href="#__codelineno-38-4"></a><span class="w"> </span>--from-literal<span class="o">=</span><span class="nv">DB_PASSWORD</span><span class="o">=</span>pass456<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-5" name="__codelineno-38-5" href="#__codelineno-38-5"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-6" name="__codelineno-38-6" href="#__codelineno-38-6"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>private/myapp-credentials.yaml
<a id="__codelineno-38-7" name="__codelineno-38-7" href="#__codelineno-38-7"></a>
<a id="__codelineno-38-8" name="__codelineno-38-8" href="#__codelineno-38-8"></a><span class="c1"># Method 2: From file</span>
<a id="__codelineno-38-9" name="__codelineno-38-9" href="#__codelineno-38-9"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>myapp-credentials<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-10" name="__codelineno-38-10" href="#__codelineno-38-10"></a><span class="w"> </span>--from-file<span class="o">=</span>.env<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-11" name="__codelineno-38-11" href="#__codelineno-38-11"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-12" name="__codelineno-38-12" href="#__codelineno-38-12"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>private/myapp-credentials.yaml
<a id="__codelineno-38-13" name="__codelineno-38-13" href="#__codelineno-38-13"></a>
<a id="__codelineno-38-14" name="__codelineno-38-14" href="#__codelineno-38-14"></a><span class="c1"># Method 3: From multiple files</span>
<a id="__codelineno-38-15" name="__codelineno-38-15" href="#__codelineno-38-15"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>myapp-credentials<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-16" name="__codelineno-38-16" href="#__codelineno-38-16"></a><span class="w"> </span>--from-file<span class="o">=</span>api-key.txt<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-17" name="__codelineno-38-17" href="#__codelineno-38-17"></a><span class="w"> </span>--from-file<span class="o">=</span>db-password.txt<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-18" name="__codelineno-38-18" href="#__codelineno-38-18"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-38-19" name="__codelineno-38-19" href="#__codelineno-38-19"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>private/myapp-credentials.yaml
</code></pre></div>
<h4 id="step-3-seal-secret">Step 3: Seal Secret<a class="headerlink" href="#step-3-seal-secret" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-39-1" name="__codelineno-39-1" href="#__codelineno-39-1"></a>kubeseal<span class="w"> </span>--format<span class="o">=</span>yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-39-2" name="__codelineno-39-2" href="#__codelineno-39-2"></a><span class="w"> </span>--cert<span class="o">=</span>pub-cert.pem<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-39-3" name="__codelineno-39-3" href="#__codelineno-39-3"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-39-4" name="__codelineno-39-4" href="#__codelineno-39-4"></a><span class="w"> </span>&lt;<span class="w"> </span>private/myapp-credentials.yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-39-5" name="__codelineno-39-5" href="#__codelineno-39-5"></a><span class="w"> </span>&gt;<span class="w"> </span>secrets/myapp-credentials-sealed.yaml
</code></pre></div>
<h4 id="step-4-commit-sealed-secret">Step 4: Commit Sealed Secret<a class="headerlink" href="#step-4-commit-sealed-secret" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-40-1" name="__codelineno-40-1" href="#__codelineno-40-1"></a>git<span class="w"> </span>add<span class="w"> </span>secrets/myapp-credentials-sealed.yaml
<a id="__codelineno-40-2" name="__codelineno-40-2" href="#__codelineno-40-2"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Add myapp credentials&quot;</span>
<a id="__codelineno-40-3" name="__codelineno-40-3" href="#__codelineno-40-3"></a>git<span class="w"> </span>push
<a id="__codelineno-40-4" name="__codelineno-40-4" href="#__codelineno-40-4"></a>
<a id="__codelineno-40-5" name="__codelineno-40-5" href="#__codelineno-40-5"></a><span class="c1"># Delete plain secret</span>
<a id="__codelineno-40-6" name="__codelineno-40-6" href="#__codelineno-40-6"></a>rm<span class="w"> </span>private/myapp-credentials.yaml
</code></pre></div>
<h3 id="updating-secrets">Updating Secrets<a class="headerlink" href="#updating-secrets" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-41-1" name="__codelineno-41-1" href="#__codelineno-41-1"></a><span class="c1"># 1. Create new version</span>
<a id="__codelineno-41-2" name="__codelineno-41-2" href="#__codelineno-41-2"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>myapp-credentials<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-3" name="__codelineno-41-3" href="#__codelineno-41-3"></a><span class="w"> </span>--from-literal<span class="o">=</span><span class="nv">API_KEY</span><span class="o">=</span>new-secret-key<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-4" name="__codelineno-41-4" href="#__codelineno-41-4"></a><span class="w"> </span>--from-literal<span class="o">=</span><span class="nv">DB_PASSWORD</span><span class="o">=</span>new-password<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-5" name="__codelineno-41-5" href="#__codelineno-41-5"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-6" name="__codelineno-41-6" href="#__codelineno-41-6"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>private/myapp-credentials.yaml
<a id="__codelineno-41-7" name="__codelineno-41-7" href="#__codelineno-41-7"></a>
<a id="__codelineno-41-8" name="__codelineno-41-8" href="#__codelineno-41-8"></a><span class="c1"># 2. Seal it</span>
<a id="__codelineno-41-9" name="__codelineno-41-9" href="#__codelineno-41-9"></a>kubeseal<span class="w"> </span>--format<span class="o">=</span>yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-10" name="__codelineno-41-10" href="#__codelineno-41-10"></a><span class="w"> </span>--cert<span class="o">=</span>pub-cert.pem<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-11" name="__codelineno-41-11" href="#__codelineno-41-11"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-12" name="__codelineno-41-12" href="#__codelineno-41-12"></a><span class="w"> </span>&lt;<span class="w"> </span>private/myapp-credentials.yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-41-13" name="__codelineno-41-13" href="#__codelineno-41-13"></a><span class="w"> </span>&gt;<span class="w"> </span>secrets/myapp-credentials-sealed.yaml
<a id="__codelineno-41-14" name="__codelineno-41-14" href="#__codelineno-41-14"></a>
<a id="__codelineno-41-15" name="__codelineno-41-15" href="#__codelineno-41-15"></a><span class="c1"># 3. Commit</span>
<a id="__codelineno-41-16" name="__codelineno-41-16" href="#__codelineno-41-16"></a>git<span class="w"> </span>add<span class="w"> </span>secrets/myapp-credentials-sealed.yaml
<a id="__codelineno-41-17" name="__codelineno-41-17" href="#__codelineno-41-17"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Update myapp credentials&quot;</span>
<a id="__codelineno-41-18" name="__codelineno-41-18" href="#__codelineno-41-18"></a>git<span class="w"> </span>push
<a id="__codelineno-41-19" name="__codelineno-41-19" href="#__codelineno-41-19"></a>
<a id="__codelineno-41-20" name="__codelineno-41-20" href="#__codelineno-41-20"></a><span class="c1"># 4. Restart pods to pick up new secret</span>
<a id="__codelineno-41-21" name="__codelineno-41-21" href="#__codelineno-41-21"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>restart<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-41-22" name="__codelineno-41-22" href="#__codelineno-41-22"></a>
<a id="__codelineno-41-23" name="__codelineno-41-23" href="#__codelineno-41-23"></a><span class="c1"># 5. Delete plain secret</span>
<a id="__codelineno-41-24" name="__codelineno-41-24" href="#__codelineno-41-24"></a>rm<span class="w"> </span>private/myapp-credentials.yaml
</code></pre></div>
<h3 id="viewing-secrets-unsealed">Viewing Secrets (Unsealed)<a class="headerlink" href="#viewing-secrets-unsealed" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-42-1" name="__codelineno-42-1" href="#__codelineno-42-1"></a><span class="c1"># List secrets in namespace</span>
<a id="__codelineno-42-2" name="__codelineno-42-2" href="#__codelineno-42-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secrets<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-42-3" name="__codelineno-42-3" href="#__codelineno-42-3"></a>
<a id="__codelineno-42-4" name="__codelineno-42-4" href="#__codelineno-42-4"></a><span class="c1"># Describe secret (doesn&#39;t show values)</span>
<a id="__codelineno-42-5" name="__codelineno-42-5" href="#__codelineno-42-5"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>secret<span class="w"> </span>myapp-credentials<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-42-6" name="__codelineno-42-6" href="#__codelineno-42-6"></a>
<a id="__codelineno-42-7" name="__codelineno-42-7" href="#__codelineno-42-7"></a><span class="c1"># View secret values (base64 encoded)</span>
<a id="__codelineno-42-8" name="__codelineno-42-8" href="#__codelineno-42-8"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>myapp-credentials<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-o<span class="w"> </span>yaml
<a id="__codelineno-42-9" name="__codelineno-42-9" href="#__codelineno-42-9"></a>
<a id="__codelineno-42-10" name="__codelineno-42-10" href="#__codelineno-42-10"></a><span class="c1"># Decode secret value</span>
<a id="__codelineno-42-11" name="__codelineno-42-11" href="#__codelineno-42-11"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>myapp-credentials<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-42-12" name="__codelineno-42-12" href="#__codelineno-42-12"></a><span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.data.API_KEY}&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>base64<span class="w"> </span>-d
</code></pre></div>
<h3 id="secret-cloning-kyverno">Secret Cloning (Kyverno)<a class="headerlink" href="#secret-cloning-kyverno" title="Permanent link">&para;</a></h3>
<p>Secrets labeled <code>allowedToBeCloned: "true"</code> in the <code>secrets</code> namespace are automatically cloned to new namespaces.</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-43-1" name="__codelineno-43-1" href="#__codelineno-43-1"></a><span class="c1"># Example: secrets-namespace.yaml</span>
<a id="__codelineno-43-2" name="__codelineno-43-2" href="#__codelineno-43-2"></a><span class="nt">apiVersion</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">v1</span>
<a id="__codelineno-43-3" name="__codelineno-43-3" href="#__codelineno-43-3"></a><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Secret</span>
<a id="__codelineno-43-4" name="__codelineno-43-4" href="#__codelineno-43-4"></a><span class="nt">metadata</span><span class="p">:</span>
<a id="__codelineno-43-5" name="__codelineno-43-5" href="#__codelineno-43-5"></a><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">shared-credentials</span>
<a id="__codelineno-43-6" name="__codelineno-43-6" href="#__codelineno-43-6"></a><span class="w"> </span><span class="nt">namespace</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">secrets</span>
<a id="__codelineno-43-7" name="__codelineno-43-7" href="#__codelineno-43-7"></a><span class="w"> </span><span class="nt">labels</span><span class="p">:</span>
<a id="__codelineno-43-8" name="__codelineno-43-8" href="#__codelineno-43-8"></a><span class="w"> </span><span class="nt">allowedToBeCloned</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;true&quot;</span>
<a id="__codelineno-43-9" name="__codelineno-43-9" href="#__codelineno-43-9"></a><span class="nt">type</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">Opaque</span>
<a id="__codelineno-43-10" name="__codelineno-43-10" href="#__codelineno-43-10"></a><span class="nt">data</span><span class="p">:</span>
<a id="__codelineno-43-11" name="__codelineno-43-11" href="#__codelineno-43-11"></a><span class="w"> </span><span class="nt">API_KEY</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">&lt;base64-encoded-value&gt;</span>
</code></pre></div>
<p>When a new namespace is created, Kyverno automatically copies this secret.</p>
<h3 id="authentication-secrets">Authentication Secrets<a class="headerlink" href="#authentication-secrets" title="Permanent link">&para;</a></h3>
<p>Applications using the authentication sidecar require specific secrets depending on the auth mode.</p>
<h4 id="token-mode-secrets">Token Mode Secrets<a class="headerlink" href="#token-mode-secrets" title="Permanent link">&para;</a></h4>
<p>Token-based auth uses an <code>auth-tokens</code> Secret:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-44-1" name="__codelineno-44-1" href="#__codelineno-44-1"></a><span class="c1"># Method 1: From Helm values (automatic)</span>
<a id="__codelineno-44-2" name="__codelineno-44-2" href="#__codelineno-44-2"></a><span class="c1"># Tokens specified in values.yaml are automatically created</span>
<a id="__codelineno-44-3" name="__codelineno-44-3" href="#__codelineno-44-3"></a>
<a id="__codelineno-44-4" name="__codelineno-44-4" href="#__codelineno-44-4"></a><span class="c1"># Method 2: Manual creation</span>
<a id="__codelineno-44-5" name="__codelineno-44-5" href="#__codelineno-44-5"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>auth-tokens<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-44-6" name="__codelineno-44-6" href="#__codelineno-44-6"></a><span class="w"> </span>--from-literal<span class="o">=</span><span class="nv">tokens</span><span class="o">=</span><span class="s2">&quot;token1</span>
<a id="__codelineno-44-7" name="__codelineno-44-7" href="#__codelineno-44-7"></a><span class="s2">token2</span>
<a id="__codelineno-44-8" name="__codelineno-44-8" href="#__codelineno-44-8"></a><span class="s2">token3&quot;</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-44-9" name="__codelineno-44-9" href="#__codelineno-44-9"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp
<a id="__codelineno-44-10" name="__codelineno-44-10" href="#__codelineno-44-10"></a>
<a id="__codelineno-44-11" name="__codelineno-44-11" href="#__codelineno-44-11"></a><span class="c1"># Method 3: From file</span>
<a id="__codelineno-44-12" name="__codelineno-44-12" href="#__codelineno-44-12"></a><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;d4f88f6d9292c10cc3e21c4aad56d2be485db532b54fe961d738e1137d247823&quot;</span><span class="w"> </span>&gt;<span class="w"> </span>tokens.txt
<a id="__codelineno-44-13" name="__codelineno-44-13" href="#__codelineno-44-13"></a><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;8803f621acc3898df1d7a8f514bc3602551a0681a8f747bd4e43c3c5849d57a7&quot;</span><span class="w"> </span>&gt;&gt;<span class="w"> </span>tokens.txt
<a id="__codelineno-44-14" name="__codelineno-44-14" href="#__codelineno-44-14"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>auth-tokens<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-44-15" name="__codelineno-44-15" href="#__codelineno-44-15"></a><span class="w"> </span>--from-file<span class="o">=</span><span class="nv">tokens</span><span class="o">=</span>tokens.txt<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-44-16" name="__codelineno-44-16" href="#__codelineno-44-16"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp
<a id="__codelineno-44-17" name="__codelineno-44-17" href="#__codelineno-44-17"></a>rm<span class="w"> </span>tokens.txt
</code></pre></div>
<h4 id="oidc-mode-secrets">OIDC Mode Secrets<a class="headerlink" href="#oidc-mode-secrets" title="Permanent link">&para;</a></h4>
<p>OIDC auth requires an <code>auth-oidc</code> Secret with two keys:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-45-1" name="__codelineno-45-1" href="#__codelineno-45-1"></a><span class="c1"># Generate secrets</span>
<a id="__codelineno-45-2" name="__codelineno-45-2" href="#__codelineno-45-2"></a><span class="nv">CLIENT_SECRET</span><span class="o">=</span><span class="s2">&quot;your-oidc-client-secret-from-provider&quot;</span>
<a id="__codelineno-45-3" name="__codelineno-45-3" href="#__codelineno-45-3"></a><span class="nv">COOKIE_SECRET</span><span class="o">=</span><span class="k">$(</span>openssl<span class="w"> </span>rand<span class="w"> </span>-hex<span class="w"> </span><span class="m">32</span><span class="k">)</span>
<a id="__codelineno-45-4" name="__codelineno-45-4" href="#__codelineno-45-4"></a>
<a id="__codelineno-45-5" name="__codelineno-45-5" href="#__codelineno-45-5"></a><span class="c1"># Create plain secret</span>
<a id="__codelineno-45-6" name="__codelineno-45-6" href="#__codelineno-45-6"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>auth-oidc<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-7" name="__codelineno-45-7" href="#__codelineno-45-7"></a><span class="w"> </span>--from-literal<span class="o">=</span>client-secret<span class="o">=</span><span class="nv">$CLIENT_SECRET</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-8" name="__codelineno-45-8" href="#__codelineno-45-8"></a><span class="w"> </span>--from-literal<span class="o">=</span>cookie-secret<span class="o">=</span><span class="nv">$COOKIE_SECRET</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-9" name="__codelineno-45-9" href="#__codelineno-45-9"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-10" name="__codelineno-45-10" href="#__codelineno-45-10"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>private/myapp-auth-oidc.yaml
<a id="__codelineno-45-11" name="__codelineno-45-11" href="#__codelineno-45-11"></a>
<a id="__codelineno-45-12" name="__codelineno-45-12" href="#__codelineno-45-12"></a><span class="c1"># Seal it</span>
<a id="__codelineno-45-13" name="__codelineno-45-13" href="#__codelineno-45-13"></a>kubeseal<span class="w"> </span>--format<span class="o">=</span>yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-14" name="__codelineno-45-14" href="#__codelineno-45-14"></a><span class="w"> </span>--cert<span class="o">=</span>pub-cert.pem<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-15" name="__codelineno-45-15" href="#__codelineno-45-15"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-16" name="__codelineno-45-16" href="#__codelineno-45-16"></a><span class="w"> </span>&lt;<span class="w"> </span>private/myapp-auth-oidc.yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-45-17" name="__codelineno-45-17" href="#__codelineno-45-17"></a><span class="w"> </span>&gt;<span class="w"> </span>secrets/myapp-auth-oidc-sealed.yaml
<a id="__codelineno-45-18" name="__codelineno-45-18" href="#__codelineno-45-18"></a>
<a id="__codelineno-45-19" name="__codelineno-45-19" href="#__codelineno-45-19"></a><span class="c1"># Apply sealed secret</span>
<a id="__codelineno-45-20" name="__codelineno-45-20" href="#__codelineno-45-20"></a>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>secrets/myapp-auth-oidc-sealed.yaml
<a id="__codelineno-45-21" name="__codelineno-45-21" href="#__codelineno-45-21"></a>
<a id="__codelineno-45-22" name="__codelineno-45-22" href="#__codelineno-45-22"></a><span class="c1"># Commit to Git</span>
<a id="__codelineno-45-23" name="__codelineno-45-23" href="#__codelineno-45-23"></a>git<span class="w"> </span>add<span class="w"> </span>secrets/myapp-auth-oidc-sealed.yaml
<a id="__codelineno-45-24" name="__codelineno-45-24" href="#__codelineno-45-24"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Add OIDC secrets for myapp&quot;</span>
<a id="__codelineno-45-25" name="__codelineno-45-25" href="#__codelineno-45-25"></a>git<span class="w"> </span>push
<a id="__codelineno-45-26" name="__codelineno-45-26" href="#__codelineno-45-26"></a>
<a id="__codelineno-45-27" name="__codelineno-45-27" href="#__codelineno-45-27"></a><span class="c1"># Clean up</span>
<a id="__codelineno-45-28" name="__codelineno-45-28" href="#__codelineno-45-28"></a>rm<span class="w"> </span>private/myapp-auth-oidc.yaml
</code></pre></div>
<h4 id="rotating-authentication-secrets">Rotating Authentication Secrets<a class="headerlink" href="#rotating-authentication-secrets" title="Permanent link">&para;</a></h4>
<p><strong>Token Rotation</strong>:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-46-1" name="__codelineno-46-1" href="#__codelineno-46-1"></a><span class="c1"># Generate new token</span>
<a id="__codelineno-46-2" name="__codelineno-46-2" href="#__codelineno-46-2"></a><span class="nv">NEW_TOKEN</span><span class="o">=</span><span class="k">$(</span>openssl<span class="w"> </span>rand<span class="w"> </span>-hex<span class="w"> </span><span class="m">32</span><span class="k">)</span>
<a id="__codelineno-46-3" name="__codelineno-46-3" href="#__codelineno-46-3"></a>
<a id="__codelineno-46-4" name="__codelineno-46-4" href="#__codelineno-46-4"></a><span class="c1"># Get current tokens</span>
<a id="__codelineno-46-5" name="__codelineno-46-5" href="#__codelineno-46-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>auth-tokens<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>/tmp/tokens.yaml
<a id="__codelineno-46-6" name="__codelineno-46-6" href="#__codelineno-46-6"></a>
<a id="__codelineno-46-7" name="__codelineno-46-7" href="#__codelineno-46-7"></a><span class="c1"># Edit tokens (add new, optionally remove old)</span>
<a id="__codelineno-46-8" name="__codelineno-46-8" href="#__codelineno-46-8"></a><span class="c1"># Then re-seal and apply</span>
<a id="__codelineno-46-9" name="__codelineno-46-9" href="#__codelineno-46-9"></a>
<a id="__codelineno-46-10" name="__codelineno-46-10" href="#__codelineno-46-10"></a><span class="c1"># Restart pods to use new tokens</span>
<a id="__codelineno-46-11" name="__codelineno-46-11" href="#__codelineno-46-11"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>restart<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp
</code></pre></div>
<p><strong>OIDC Secret Rotation</strong>:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-47-1" name="__codelineno-47-1" href="#__codelineno-47-1"></a><span class="c1"># Rotate cookie secret (safe - invalidates existing sessions)</span>
<a id="__codelineno-47-2" name="__codelineno-47-2" href="#__codelineno-47-2"></a><span class="nv">NEW_COOKIE_SECRET</span><span class="o">=</span><span class="k">$(</span>openssl<span class="w"> </span>rand<span class="w"> </span>-hex<span class="w"> </span><span class="m">32</span><span class="k">)</span>
<a id="__codelineno-47-3" name="__codelineno-47-3" href="#__codelineno-47-3"></a>
<a id="__codelineno-47-4" name="__codelineno-47-4" href="#__codelineno-47-4"></a><span class="c1"># Recreate secret</span>
<a id="__codelineno-47-5" name="__codelineno-47-5" href="#__codelineno-47-5"></a>kubectl<span class="w"> </span>create<span class="w"> </span>secret<span class="w"> </span>generic<span class="w"> </span>auth-oidc<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-47-6" name="__codelineno-47-6" href="#__codelineno-47-6"></a><span class="w"> </span>--from-literal<span class="o">=</span>client-secret<span class="o">=</span><span class="nv">$CLIENT_SECRET</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-47-7" name="__codelineno-47-7" href="#__codelineno-47-7"></a><span class="w"> </span>--from-literal<span class="o">=</span>cookie-secret<span class="o">=</span><span class="nv">$NEW_COOKIE_SECRET</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-47-8" name="__codelineno-47-8" href="#__codelineno-47-8"></a><span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-47-9" name="__codelineno-47-9" href="#__codelineno-47-9"></a><span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-47-10" name="__codelineno-47-10" href="#__codelineno-47-10"></a><span class="w"> </span>kubeseal<span class="w"> </span>--format<span class="o">=</span>yaml<span class="w"> </span>--cert<span class="o">=</span>pub-cert.pem<span class="w"> </span>--namespace<span class="o">=</span>myapp<span class="w"> </span><span class="p">|</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-47-11" name="__codelineno-47-11" href="#__codelineno-47-11"></a><span class="w"> </span>kubectl<span class="w"> </span>apply<span class="w"> </span>-f<span class="w"> </span>-
<a id="__codelineno-47-12" name="__codelineno-47-12" href="#__codelineno-47-12"></a>
<a id="__codelineno-47-13" name="__codelineno-47-13" href="#__codelineno-47-13"></a><span class="c1"># Restart to pick up new secret</span>
<a id="__codelineno-47-14" name="__codelineno-47-14" href="#__codelineno-47-14"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>restart<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp
</code></pre></div>
<h4 id="viewing-authentication-secrets">Viewing Authentication Secrets<a class="headerlink" href="#viewing-authentication-secrets" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-48-1" name="__codelineno-48-1" href="#__codelineno-48-1"></a><span class="c1"># List auth-related secrets</span>
<a id="__codelineno-48-2" name="__codelineno-48-2" href="#__codelineno-48-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secrets<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>auth
<a id="__codelineno-48-3" name="__codelineno-48-3" href="#__codelineno-48-3"></a>
<a id="__codelineno-48-4" name="__codelineno-48-4" href="#__codelineno-48-4"></a><span class="c1"># View token secret (tokens are in plain text in the Secret)</span>
<a id="__codelineno-48-5" name="__codelineno-48-5" href="#__codelineno-48-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>auth-tokens<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.data.tokens}&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>base64<span class="w"> </span>-d
<a id="__codelineno-48-6" name="__codelineno-48-6" href="#__codelineno-48-6"></a>
<a id="__codelineno-48-7" name="__codelineno-48-7" href="#__codelineno-48-7"></a><span class="c1"># View OIDC secret keys (values are base64 encoded)</span>
<a id="__codelineno-48-8" name="__codelineno-48-8" href="#__codelineno-48-8"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>auth-oidc<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.data.client-secret}&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>base64<span class="w"> </span>-d
<a id="__codelineno-48-9" name="__codelineno-48-9" href="#__codelineno-48-9"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>auth-oidc<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.data.cookie-secret}&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>base64<span class="w"> </span>-d
</code></pre></div>
<p><strong>See</strong>: <a href="../docs/DEVELOPER-GUIDE.md#enabling-authentication-for-applications">Developer Guide - Enabling Authentication</a> for complete authentication setup guide.</p>
<hr />
<h2 id="monitoring-alerting">Monitoring &amp; Alerting<a class="headerlink" href="#monitoring-alerting" title="Permanent link">&para;</a></h2>
<h3 id="prometheus-metrics">Prometheus Metrics<a class="headerlink" href="#prometheus-metrics" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-49-1" name="__codelineno-49-1" href="#__codelineno-49-1"></a><span class="c1"># Port forward to Prometheus</span>
<a id="__codelineno-49-2" name="__codelineno-49-2" href="#__codelineno-49-2"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>svc/prometheus-server<span class="w"> </span><span class="m">9090</span>:80
<a id="__codelineno-49-3" name="__codelineno-49-3" href="#__codelineno-49-3"></a>
<a id="__codelineno-49-4" name="__codelineno-49-4" href="#__codelineno-49-4"></a><span class="c1"># Access: http://localhost:9090</span>
</code></pre></div>
<p><strong>Common Queries:</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-50-1" name="__codelineno-50-1" href="#__codelineno-50-1"></a><span class="c1"># CPU usage per pod</span>
<a id="__codelineno-50-2" name="__codelineno-50-2" href="#__codelineno-50-2"></a><span class="k">sum</span><span class="o">(</span><span class="kr">rate</span><span class="o">(</span><span class="nv">container_cpu_usage_seconds_total</span><span class="p">[</span><span class="s">5m</span><span class="p">]</span><span class="o">))</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">pod</span><span class="o">)</span>
<a id="__codelineno-50-3" name="__codelineno-50-3" href="#__codelineno-50-3"></a>
<a id="__codelineno-50-4" name="__codelineno-50-4" href="#__codelineno-50-4"></a><span class="c1"># Memory usage per pod</span>
<a id="__codelineno-50-5" name="__codelineno-50-5" href="#__codelineno-50-5"></a><span class="k">sum</span><span class="o">(</span><span class="nv">container_memory_usage_bytes</span><span class="o">)</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">pod</span><span class="o">)</span>
<a id="__codelineno-50-6" name="__codelineno-50-6" href="#__codelineno-50-6"></a>
<a id="__codelineno-50-7" name="__codelineno-50-7" href="#__codelineno-50-7"></a><span class="c1"># Request rate per service</span>
<a id="__codelineno-50-8" name="__codelineno-50-8" href="#__codelineno-50-8"></a><span class="kr">rate</span><span class="o">(</span><span class="nv">http_requests_total</span><span class="p">[</span><span class="s">5m</span><span class="p">]</span><span class="o">)</span>
</code></pre></div></p>
<h3 id="grafana-dashboards">Grafana Dashboards<a class="headerlink" href="#grafana-dashboards" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-51-1" name="__codelineno-51-1" href="#__codelineno-51-1"></a><span class="c1"># Port forward to Grafana</span>
<a id="__codelineno-51-2" name="__codelineno-51-2" href="#__codelineno-51-2"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>svc/grafana<span class="w"> </span><span class="m">3000</span>:80
<a id="__codelineno-51-3" name="__codelineno-51-3" href="#__codelineno-51-3"></a>
<a id="__codelineno-51-4" name="__codelineno-51-4" href="#__codelineno-51-4"></a><span class="c1"># Access: http://localhost:3000</span>
</code></pre></div>
<h3 id="loki-logs">Loki Logs<a class="headerlink" href="#loki-logs" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-52-1" name="__codelineno-52-1" href="#__codelineno-52-1"></a><span class="c1"># Port forward to Loki</span>
<a id="__codelineno-52-2" name="__codelineno-52-2" href="#__codelineno-52-2"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>svc/loki<span class="w"> </span><span class="m">3100</span>:3100
<a id="__codelineno-52-3" name="__codelineno-52-3" href="#__codelineno-52-3"></a>
<a id="__codelineno-52-4" name="__codelineno-52-4" href="#__codelineno-52-4"></a><span class="c1"># Query logs</span>
<a id="__codelineno-52-5" name="__codelineno-52-5" href="#__codelineno-52-5"></a>curl<span class="w"> </span>-G<span class="w"> </span>-s<span class="w"> </span><span class="s1">&#39;http://localhost:3100/loki/api/v1/query_range&#39;</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-52-6" name="__codelineno-52-6" href="#__codelineno-52-6"></a><span class="w"> </span>--data-urlencode<span class="w"> </span><span class="s1">&#39;query={namespace=&quot;myapp&quot;}&#39;</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-52-7" name="__codelineno-52-7" href="#__codelineno-52-7"></a><span class="w"> </span>--data-urlencode<span class="w"> </span><span class="s1">&#39;start=1h&#39;</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>jq
</code></pre></div>
<h3 id="tempo-traces">Tempo Traces<a class="headerlink" href="#tempo-traces" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-53-1" name="__codelineno-53-1" href="#__codelineno-53-1"></a><span class="c1"># Port forward to Tempo query API</span>
<a id="__codelineno-53-2" name="__codelineno-53-2" href="#__codelineno-53-2"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>svc/tempo<span class="w"> </span><span class="m">3200</span>:3200
<a id="__codelineno-53-3" name="__codelineno-53-3" href="#__codelineno-53-3"></a>
<a id="__codelineno-53-4" name="__codelineno-53-4" href="#__codelineno-53-4"></a><span class="c1"># Access: http://localhost:3200</span>
</code></pre></div>
<p><strong>Query traces via Grafana:</strong>
1. Open Grafana → Explore
2. Select Tempo datasource
3. Use TraceQL or search by service name</p>
<p><strong>Verify Traefik is sending traces:</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-54-1" name="__codelineno-54-1" href="#__codelineno-54-1"></a><span class="c1"># Check Traefik logs for OTLP export errors</span>
<a id="__codelineno-54-2" name="__codelineno-54-2" href="#__codelineno-54-2"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>traefik-system<span class="w"> </span>-l<span class="w"> </span>app.kubernetes.io/name<span class="o">=</span>traefik<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>-i<span class="w"> </span><span class="s2">&quot;traces export&quot;</span>
<a id="__codelineno-54-3" name="__codelineno-54-3" href="#__codelineno-54-3"></a>
<a id="__codelineno-54-4" name="__codelineno-54-4" href="#__codelineno-54-4"></a><span class="c1"># Check Tempo is receiving data</span>
<a id="__codelineno-54-5" name="__codelineno-54-5" href="#__codelineno-54-5"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>-l<span class="w"> </span>app.kubernetes.io/name<span class="o">=</span>tempo<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span><span class="s2">&quot;receiver&quot;</span>
</code></pre></div></p>
<p><strong>Trace-to-log correlation:</strong>
- Click a trace span in Grafana → linked Loki logs appear (by namespace, pod, container)
- Trace-to-metrics links to Prometheus by service name</p>
<h3 id="fluent-bit-log-shipping">Fluent-Bit Log Shipping<a class="headerlink" href="#fluent-bit-log-shipping" title="Permanent link">&para;</a></h3>
<p>Verify Fluent-Bit is shipping logs:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-55-1" name="__codelineno-55-1" href="#__codelineno-55-1"></a><span class="c1"># Check Fluent-Bit pods</span>
<a id="__codelineno-55-2" name="__codelineno-55-2" href="#__codelineno-55-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pods<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>fluent-bit
<a id="__codelineno-55-3" name="__codelineno-55-3" href="#__codelineno-55-3"></a>
<a id="__codelineno-55-4" name="__codelineno-55-4" href="#__codelineno-55-4"></a><span class="c1"># Check logs</span>
<a id="__codelineno-55-5" name="__codelineno-55-5" href="#__codelineno-55-5"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>daemonset/fluent-bit
<a id="__codelineno-55-6" name="__codelineno-55-6" href="#__codelineno-55-6"></a>
<a id="__codelineno-55-7" name="__codelineno-55-7" href="#__codelineno-55-7"></a><span class="c1"># Verify Loki is receiving logs</span>
<a id="__codelineno-55-8" name="__codelineno-55-8" href="#__codelineno-55-8"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>monitoring<span class="w"> </span>deployment/loki<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span><span class="s2">&quot;POST /loki/api/v1/push&quot;</span>
</code></pre></div>
<h3 id="trivy-vulnerability-scanning">Trivy Vulnerability Scanning<a class="headerlink" href="#trivy-vulnerability-scanning" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-56-1" name="__codelineno-56-1" href="#__codelineno-56-1"></a><span class="c1"># Check Trivy scan results</span>
<a id="__codelineno-56-2" name="__codelineno-56-2" href="#__codelineno-56-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>vulnerabilityreports<span class="w"> </span>--all-namespaces
<a id="__codelineno-56-3" name="__codelineno-56-3" href="#__codelineno-56-3"></a>
<a id="__codelineno-56-4" name="__codelineno-56-4" href="#__codelineno-56-4"></a><span class="c1"># View report for specific pod</span>
<a id="__codelineno-56-5" name="__codelineno-56-5" href="#__codelineno-56-5"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>vulnerabilityreport<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>&lt;report-name&gt;
</code></pre></div>
<h3 id="slack-notifications">Slack Notifications<a class="headerlink" href="#slack-notifications" title="Permanent link">&para;</a></h3>
<p>All applications have Slack notifications enabled:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-57-1" name="__codelineno-57-1" href="#__codelineno-57-1"></a><span class="nt">metadata</span><span class="p">:</span>
<a id="__codelineno-57-2" name="__codelineno-57-2" href="#__codelineno-57-2"></a><span class="w"> </span><span class="nt">annotations</span><span class="p">:</span>
<a id="__codelineno-57-3" name="__codelineno-57-3" href="#__codelineno-57-3"></a><span class="w"> </span><span class="nt">notifications.argoproj.io/subscribe.on-sync-succeeded.slack</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;&quot;</span>
<a id="__codelineno-57-4" name="__codelineno-57-4" href="#__codelineno-57-4"></a><span class="w"> </span><span class="nt">notifications.argoproj.io/subscribe.on-sync-failed.slack</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;&quot;</span>
<a id="__codelineno-57-5" name="__codelineno-57-5" href="#__codelineno-57-5"></a><span class="w"> </span><span class="nt">notifications.argoproj.io/subscribe.on-degraded.slack</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;&quot;</span>
</code></pre></div>
<p><strong>Test Notification:</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-58-1" name="__codelineno-58-1" href="#__codelineno-58-1"></a><span class="c1"># Trigger a sync to test</span>
<a id="__codelineno-58-2" name="__codelineno-58-2" href="#__codelineno-58-2"></a>kubectl<span class="w"> </span>patch<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-58-3" name="__codelineno-58-3" href="#__codelineno-58-3"></a><span class="w"> </span>--type<span class="w"> </span>merge<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-58-4" name="__codelineno-58-4" href="#__codelineno-58-4"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;metadata&quot;:{&quot;annotations&quot;:{&quot;argocd.argoproj.io/refresh&quot;:&quot;hard&quot;}}}&#39;</span>
</code></pre></div></p>
<hr />
<h2 id="troubleshooting">Troubleshooting<a class="headerlink" href="#troubleshooting" title="Permanent link">&para;</a></h2>
<h3 id="application-wont-sync">Application Won't Sync<a class="headerlink" href="#application-wont-sync" title="Permanent link">&para;</a></h3>
<h4 id="check-application-status">Check Application Status<a class="headerlink" href="#check-application-status" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-59-1" name="__codelineno-59-1" href="#__codelineno-59-1"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd
</code></pre></div>
<p>Look for errors in:
- <code>Status.Conditions</code>
- <code>Status.OperationState</code></p>
<h4 id="common-issues">Common Issues<a class="headerlink" href="#common-issues" title="Permanent link">&para;</a></h4>
<p><strong>Issue 1: Image Pull Error</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-60-1" name="__codelineno-60-1" href="#__codelineno-60-1"></a><span class="c1"># Error: ErrImagePull, ImagePullBackOff</span>
<a id="__codelineno-60-2" name="__codelineno-60-2" href="#__codelineno-60-2"></a>
<a id="__codelineno-60-3" name="__codelineno-60-3" href="#__codelineno-60-3"></a><span class="c1"># Check if image exists</span>
<a id="__codelineno-60-4" name="__codelineno-60-4" href="#__codelineno-60-4"></a>docker<span class="w"> </span>pull<span class="w"> </span>ghcr.io/fortedigital/myapp:v1.0.0
<a id="__codelineno-60-5" name="__codelineno-60-5" href="#__codelineno-60-5"></a>
<a id="__codelineno-60-6" name="__codelineno-60-6" href="#__codelineno-60-6"></a><span class="c1"># Check image pull secrets</span>
<a id="__codelineno-60-7" name="__codelineno-60-7" href="#__codelineno-60-7"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secrets<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>regcred
<a id="__codelineno-60-8" name="__codelineno-60-8" href="#__codelineno-60-8"></a>
<a id="__codelineno-60-9" name="__codelineno-60-9" href="#__codelineno-60-9"></a><span class="c1"># Check pod events</span>
<a id="__codelineno-60-10" name="__codelineno-60-10" href="#__codelineno-60-10"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>pod<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>&lt;pod-name&gt;
</code></pre></div></p>
<p><strong>Issue 2: Invalid YAML</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-61-1" name="__codelineno-61-1" href="#__codelineno-61-1"></a><span class="c1"># Error: unable to decode manifest</span>
<a id="__codelineno-61-2" name="__codelineno-61-2" href="#__codelineno-61-2"></a>
<a id="__codelineno-61-3" name="__codelineno-61-3" href="#__codelineno-61-3"></a><span class="c1"># Validate YAML locally</span>
<a id="__codelineno-61-4" name="__codelineno-61-4" href="#__codelineno-61-4"></a>kubectl<span class="w"> </span>apply<span class="w"> </span>--dry-run<span class="o">=</span>client<span class="w"> </span>-f<span class="w"> </span>apps/myapp.yaml
<a id="__codelineno-61-5" name="__codelineno-61-5" href="#__codelineno-61-5"></a>
<a id="__codelineno-61-6" name="__codelineno-61-6" href="#__codelineno-61-6"></a><span class="c1"># Check ArgoCD application controller logs</span>
<a id="__codelineno-61-7" name="__codelineno-61-7" href="#__codelineno-61-7"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>deployment/argocd-application-controller<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>myapp
</code></pre></div></p>
<p><strong>Issue 3: Resource Quota Exceeded</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-62-1" name="__codelineno-62-1" href="#__codelineno-62-1"></a><span class="c1"># Error: exceeded quota</span>
<a id="__codelineno-62-2" name="__codelineno-62-2" href="#__codelineno-62-2"></a>
<a id="__codelineno-62-3" name="__codelineno-62-3" href="#__codelineno-62-3"></a><span class="c1"># Check namespace quotas</span>
<a id="__codelineno-62-4" name="__codelineno-62-4" href="#__codelineno-62-4"></a>kubectl<span class="w"> </span>get<span class="w"> </span>resourcequota<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-62-5" name="__codelineno-62-5" href="#__codelineno-62-5"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>resourcequota<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-62-6" name="__codelineno-62-6" href="#__codelineno-62-6"></a>
<a id="__codelineno-62-7" name="__codelineno-62-7" href="#__codelineno-62-7"></a><span class="c1"># Increase quota or reduce resource requests</span>
</code></pre></div></p>
<h3 id="pod-crashes">Pod Crashes<a class="headerlink" href="#pod-crashes" title="Permanent link">&para;</a></h3>
<h4 id="crashloopbackoff">CrashLoopBackOff<a class="headerlink" href="#crashloopbackoff" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-63-1" name="__codelineno-63-1" href="#__codelineno-63-1"></a><span class="c1"># Check pod status</span>
<a id="__codelineno-63-2" name="__codelineno-63-2" href="#__codelineno-63-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pods<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-63-3" name="__codelineno-63-3" href="#__codelineno-63-3"></a>
<a id="__codelineno-63-4" name="__codelineno-63-4" href="#__codelineno-63-4"></a><span class="c1"># View logs</span>
<a id="__codelineno-63-5" name="__codelineno-63-5" href="#__codelineno-63-5"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>&lt;pod-name&gt;
<a id="__codelineno-63-6" name="__codelineno-63-6" href="#__codelineno-63-6"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>&lt;pod-name&gt;<span class="w"> </span>--previous<span class="w"> </span><span class="c1"># Previous container</span>
<a id="__codelineno-63-7" name="__codelineno-63-7" href="#__codelineno-63-7"></a>
<a id="__codelineno-63-8" name="__codelineno-63-8" href="#__codelineno-63-8"></a><span class="c1"># Check events</span>
<a id="__codelineno-63-9" name="__codelineno-63-9" href="#__codelineno-63-9"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>pod<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>&lt;pod-name&gt;
</code></pre></div>
<p><strong>Common Causes:</strong>
- Application error (check logs)
- Missing environment variables
- Wrong port configuration
- Missing secrets
- Insufficient memory/CPU</p>
<h4 id="imagepullbackoff">ImagePullBackOff<a class="headerlink" href="#imagepullbackoff" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-64-1" name="__codelineno-64-1" href="#__codelineno-64-1"></a><span class="c1"># Check image name</span>
<a id="__codelineno-64-2" name="__codelineno-64-2" href="#__codelineno-64-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>image
<a id="__codelineno-64-3" name="__codelineno-64-3" href="#__codelineno-64-3"></a>
<a id="__codelineno-64-4" name="__codelineno-64-4" href="#__codelineno-64-4"></a><span class="c1"># Verify credentials</span>
<a id="__codelineno-64-5" name="__codelineno-64-5" href="#__codelineno-64-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>-n<span class="w"> </span>myapp
</code></pre></div>
<h4 id="pending">Pending<a class="headerlink" href="#pending" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-65-1" name="__codelineno-65-1" href="#__codelineno-65-1"></a><span class="c1"># Check why pod is pending</span>
<a id="__codelineno-65-2" name="__codelineno-65-2" href="#__codelineno-65-2"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>pod<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>&lt;pod-name&gt;
<a id="__codelineno-65-3" name="__codelineno-65-3" href="#__codelineno-65-3"></a>
<a id="__codelineno-65-4" name="__codelineno-65-4" href="#__codelineno-65-4"></a><span class="c1"># Common reasons:</span>
<a id="__codelineno-65-5" name="__codelineno-65-5" href="#__codelineno-65-5"></a><span class="c1"># - Insufficient resources on nodes</span>
<a id="__codelineno-65-6" name="__codelineno-65-6" href="#__codelineno-65-6"></a><span class="c1"># - PVC not bound</span>
<a id="__codelineno-65-7" name="__codelineno-65-7" href="#__codelineno-65-7"></a><span class="c1"># - Node selector doesn&#39;t match</span>
</code></pre></div>
<h3 id="ingress-tls-issues">Ingress / TLS Issues<a class="headerlink" href="#ingress-tls-issues" title="Permanent link">&para;</a></h3>
<h4 id="application-not-accessible">Application Not Accessible<a class="headerlink" href="#application-not-accessible" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-66-1" name="__codelineno-66-1" href="#__codelineno-66-1"></a><span class="c1"># Check IngressRoute</span>
<a id="__codelineno-66-2" name="__codelineno-66-2" href="#__codelineno-66-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>ingressroute<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-66-3" name="__codelineno-66-3" href="#__codelineno-66-3"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>ingressroute<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-66-4" name="__codelineno-66-4" href="#__codelineno-66-4"></a>
<a id="__codelineno-66-5" name="__codelineno-66-5" href="#__codelineno-66-5"></a><span class="c1"># Check Traefik</span>
<a id="__codelineno-66-6" name="__codelineno-66-6" href="#__codelineno-66-6"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pods<span class="w"> </span>-n<span class="w"> </span>traefik
<a id="__codelineno-66-7" name="__codelineno-66-7" href="#__codelineno-66-7"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>traefik<span class="w"> </span>deployment/traefik
<a id="__codelineno-66-8" name="__codelineno-66-8" href="#__codelineno-66-8"></a>
<a id="__codelineno-66-9" name="__codelineno-66-9" href="#__codelineno-66-9"></a><span class="c1"># Test with port-forward</span>
<a id="__codelineno-66-10" name="__codelineno-66-10" href="#__codelineno-66-10"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>service/myapp<span class="w"> </span><span class="m">8080</span>:3000
<a id="__codelineno-66-11" name="__codelineno-66-11" href="#__codelineno-66-11"></a>curl<span class="w"> </span>http://localhost:8080
</code></pre></div>
<h4 id="certificate-issues">Certificate Issues<a class="headerlink" href="#certificate-issues" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-67-1" name="__codelineno-67-1" href="#__codelineno-67-1"></a><span class="c1"># Check certificates</span>
<a id="__codelineno-67-2" name="__codelineno-67-2" href="#__codelineno-67-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>certificate<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-67-3" name="__codelineno-67-3" href="#__codelineno-67-3"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>certificate<span class="w"> </span>myapp-tls<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-67-4" name="__codelineno-67-4" href="#__codelineno-67-4"></a>
<a id="__codelineno-67-5" name="__codelineno-67-5" href="#__codelineno-67-5"></a><span class="c1"># Check cert-manager</span>
<a id="__codelineno-67-6" name="__codelineno-67-6" href="#__codelineno-67-6"></a>kubectl<span class="w"> </span>get<span class="w"> </span>clusterissuer
<a id="__codelineno-67-7" name="__codelineno-67-7" href="#__codelineno-67-7"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>cert-manager<span class="w"> </span>deployment/cert-manager
<a id="__codelineno-67-8" name="__codelineno-67-8" href="#__codelineno-67-8"></a>
<a id="__codelineno-67-9" name="__codelineno-67-9" href="#__codelineno-67-9"></a><span class="c1"># Check Let&#39;s Encrypt challenges</span>
<a id="__codelineno-67-10" name="__codelineno-67-10" href="#__codelineno-67-10"></a>kubectl<span class="w"> </span>get<span class="w"> </span>challenges<span class="w"> </span>--all-namespaces
</code></pre></div>
<p><strong>Manual Certificate Renewal:</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-68-1" name="__codelineno-68-1" href="#__codelineno-68-1"></a><span class="c1"># Delete and recreate certificate</span>
<a id="__codelineno-68-2" name="__codelineno-68-2" href="#__codelineno-68-2"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>certificate<span class="w"> </span>myapp-tls<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-68-3" name="__codelineno-68-3" href="#__codelineno-68-3"></a>
<a id="__codelineno-68-4" name="__codelineno-68-4" href="#__codelineno-68-4"></a><span class="c1"># Certificate will be automatically recreated</span>
</code></pre></div></p>
<h3 id="database-issues">Database Issues<a class="headerlink" href="#database-issues" title="Permanent link">&para;</a></h3>
<h4 id="postgresql-wont-start">PostgreSQL Won't Start<a class="headerlink" href="#postgresql-wont-start" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-69-1" name="__codelineno-69-1" href="#__codelineno-69-1"></a><span class="c1"># Check StatefulSet</span>
<a id="__codelineno-69-2" name="__codelineno-69-2" href="#__codelineno-69-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>statefulset<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-69-3" name="__codelineno-69-3" href="#__codelineno-69-3"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>statefulset<span class="w"> </span>postgres<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-69-4" name="__codelineno-69-4" href="#__codelineno-69-4"></a>
<a id="__codelineno-69-5" name="__codelineno-69-5" href="#__codelineno-69-5"></a><span class="c1"># Check PVC</span>
<a id="__codelineno-69-6" name="__codelineno-69-6" href="#__codelineno-69-6"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pvc<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-69-7" name="__codelineno-69-7" href="#__codelineno-69-7"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>pvc<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-69-8" name="__codelineno-69-8" href="#__codelineno-69-8"></a>
<a id="__codelineno-69-9" name="__codelineno-69-9" href="#__codelineno-69-9"></a><span class="c1"># Check logs</span>
<a id="__codelineno-69-10" name="__codelineno-69-10" href="#__codelineno-69-10"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>postgres-0
</code></pre></div>
<h4 id="data-persistence">Data Persistence<a class="headerlink" href="#data-persistence" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-70-1" name="__codelineno-70-1" href="#__codelineno-70-1"></a><span class="c1"># Verify PVC is bound</span>
<a id="__codelineno-70-2" name="__codelineno-70-2" href="#__codelineno-70-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pvc<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-70-3" name="__codelineno-70-3" href="#__codelineno-70-3"></a>
<a id="__codelineno-70-4" name="__codelineno-70-4" href="#__codelineno-70-4"></a><span class="c1"># Check storage class</span>
<a id="__codelineno-70-5" name="__codelineno-70-5" href="#__codelineno-70-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>storageclass
<a id="__codelineno-70-6" name="__codelineno-70-6" href="#__codelineno-70-6"></a>
<a id="__codelineno-70-7" name="__codelineno-70-7" href="#__codelineno-70-7"></a><span class="c1"># Resize PVC (if supported)</span>
<a id="__codelineno-70-8" name="__codelineno-70-8" href="#__codelineno-70-8"></a>kubectl<span class="w"> </span>edit<span class="w"> </span>pvc<span class="w"> </span>postgres-data-postgres-0<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-70-9" name="__codelineno-70-9" href="#__codelineno-70-9"></a><span class="c1"># Change: storage: 10Gi (from 5Gi)</span>
</code></pre></div>
<h3 id="kyverno-policy-issues">Kyverno Policy Issues<a class="headerlink" href="#kyverno-policy-issues" title="Permanent link">&para;</a></h3>
<h4 id="policy-violations">Policy Violations<a class="headerlink" href="#policy-violations" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-71-1" name="__codelineno-71-1" href="#__codelineno-71-1"></a><span class="c1"># List policies</span>
<a id="__codelineno-71-2" name="__codelineno-71-2" href="#__codelineno-71-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>clusterpolicy
<a id="__codelineno-71-3" name="__codelineno-71-3" href="#__codelineno-71-3"></a>
<a id="__codelineno-71-4" name="__codelineno-71-4" href="#__codelineno-71-4"></a><span class="c1"># Check policy reports</span>
<a id="__codelineno-71-5" name="__codelineno-71-5" href="#__codelineno-71-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>policyreport<span class="w"> </span>--all-namespaces
<a id="__codelineno-71-6" name="__codelineno-71-6" href="#__codelineno-71-6"></a>
<a id="__codelineno-71-7" name="__codelineno-71-7" href="#__codelineno-71-7"></a><span class="c1"># View specific policy</span>
<a id="__codelineno-71-8" name="__codelineno-71-8" href="#__codelineno-71-8"></a>kubectl<span class="w"> </span>describe<span class="w"> </span>clusterpolicy<span class="w"> </span>secret-cloner
</code></pre></div>
<h4 id="secret-not-cloned">Secret Not Cloned<a class="headerlink" href="#secret-not-cloned" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-72-1" name="__codelineno-72-1" href="#__codelineno-72-1"></a><span class="c1"># Check if secret has label</span>
<a id="__codelineno-72-2" name="__codelineno-72-2" href="#__codelineno-72-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>-n<span class="w"> </span>secrets<span class="w"> </span>--show-labels
<a id="__codelineno-72-3" name="__codelineno-72-3" href="#__codelineno-72-3"></a>
<a id="__codelineno-72-4" name="__codelineno-72-4" href="#__codelineno-72-4"></a><span class="c1"># Check Kyverno logs</span>
<a id="__codelineno-72-5" name="__codelineno-72-5" href="#__codelineno-72-5"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>kyverno<span class="w"> </span>deployment/kyverno
<a id="__codelineno-72-6" name="__codelineno-72-6" href="#__codelineno-72-6"></a>
<a id="__codelineno-72-7" name="__codelineno-72-7" href="#__codelineno-72-7"></a><span class="c1"># Manually trigger by recreating namespace</span>
<a id="__codelineno-72-8" name="__codelineno-72-8" href="#__codelineno-72-8"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>ns<span class="w"> </span>test-ns
<a id="__codelineno-72-9" name="__codelineno-72-9" href="#__codelineno-72-9"></a>kubectl<span class="w"> </span>create<span class="w"> </span>ns<span class="w"> </span>test-ns
</code></pre></div>
<h3 id="argocd-issues">ArgoCD Issues<a class="headerlink" href="#argocd-issues" title="Permanent link">&para;</a></h3>
<h4 id="argocd-ui-not-accessible">ArgoCD UI Not Accessible<a class="headerlink" href="#argocd-ui-not-accessible" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-73-1" name="__codelineno-73-1" href="#__codelineno-73-1"></a><span class="c1"># Check ArgoCD pods</span>
<a id="__codelineno-73-2" name="__codelineno-73-2" href="#__codelineno-73-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pods<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-73-3" name="__codelineno-73-3" href="#__codelineno-73-3"></a>
<a id="__codelineno-73-4" name="__codelineno-73-4" href="#__codelineno-73-4"></a><span class="c1"># Restart ArgoCD server</span>
<a id="__codelineno-73-5" name="__codelineno-73-5" href="#__codelineno-73-5"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>restart<span class="w"> </span>deployment<span class="w"> </span>argocd-server<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-73-6" name="__codelineno-73-6" href="#__codelineno-73-6"></a>
<a id="__codelineno-73-7" name="__codelineno-73-7" href="#__codelineno-73-7"></a><span class="c1"># Port forward</span>
<a id="__codelineno-73-8" name="__codelineno-73-8" href="#__codelineno-73-8"></a>kubectl<span class="w"> </span>port-forward<span class="w"> </span>svc/argocd-server<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="m">8080</span>:443
</code></pre></div>
<h4 id="sync-takes-too-long">Sync Takes Too Long<a class="headerlink" href="#sync-takes-too-long" title="Permanent link">&para;</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-74-1" name="__codelineno-74-1" href="#__codelineno-74-1"></a><span class="c1"># Check application controller logs</span>
<a id="__codelineno-74-2" name="__codelineno-74-2" href="#__codelineno-74-2"></a>kubectl<span class="w"> </span>logs<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>deployment/argocd-application-controller
<a id="__codelineno-74-3" name="__codelineno-74-3" href="#__codelineno-74-3"></a>
<a id="__codelineno-74-4" name="__codelineno-74-4" href="#__codelineno-74-4"></a><span class="c1"># Increase timeout (in apps/myapp.yaml)</span>
<a id="__codelineno-74-5" name="__codelineno-74-5" href="#__codelineno-74-5"></a>spec:
<a id="__codelineno-74-6" name="__codelineno-74-6" href="#__codelineno-74-6"></a><span class="w"> </span>syncPolicy:
<a id="__codelineno-74-7" name="__codelineno-74-7" href="#__codelineno-74-7"></a><span class="w"> </span>retry:
<a id="__codelineno-74-8" name="__codelineno-74-8" href="#__codelineno-74-8"></a><span class="w"> </span>backoff:
<a id="__codelineno-74-9" name="__codelineno-74-9" href="#__codelineno-74-9"></a><span class="w"> </span>maxDuration:<span class="w"> </span>5m<span class="w"> </span><span class="c1"># Increase from 3m</span>
</code></pre></div>
<hr />
<h2 id="disaster-recovery">Disaster Recovery<a class="headerlink" href="#disaster-recovery" title="Permanent link">&para;</a></h2>
<h3 id="backup-strategy">Backup Strategy<a class="headerlink" href="#backup-strategy" title="Permanent link">&para;</a></h3>
<p><strong>Current State</strong>: No automated backups</p>
<p><strong>What Needs Backup</strong>:
- ❌ Cluster state (not backed up - recreate via GitOps)
- ❌ Persistent volumes (currently not critical)
- ✅ Git repositories (GitHub provides backup)
- ⚠️ Secrets (sealed secrets in Git, unseal keys need safekeeping)</p>
<h3 id="cluster-rebuild">Cluster Rebuild<a class="headerlink" href="#cluster-rebuild" title="Permanent link">&para;</a></h3>
<p><strong>Scenario</strong>: Complete cluster failure</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-75-1" name="__codelineno-75-1" href="#__codelineno-75-1"></a><span class="c1"># 1. Provision new Kubernetes cluster</span>
<a id="__codelineno-75-2" name="__codelineno-75-2" href="#__codelineno-75-2"></a>
<a id="__codelineno-75-3" name="__codelineno-75-3" href="#__codelineno-75-3"></a><span class="c1"># 2. Configure kubectl</span>
<a id="__codelineno-75-4" name="__codelineno-75-4" href="#__codelineno-75-4"></a>kubectl<span class="w"> </span>config<span class="w"> </span>use-context<span class="w"> </span>new-cluster
<a id="__codelineno-75-5" name="__codelineno-75-5" href="#__codelineno-75-5"></a>kubectl<span class="w"> </span>cluster-info
<a id="__codelineno-75-6" name="__codelineno-75-6" href="#__codelineno-75-6"></a>
<a id="__codelineno-75-7" name="__codelineno-75-7" href="#__codelineno-75-7"></a><span class="c1"># 3. Bootstrap cluster</span>
<a id="__codelineno-75-8" name="__codelineno-75-8" href="#__codelineno-75-8"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/launchpad
<a id="__codelineno-75-9" name="__codelineno-75-9" href="#__codelineno-75-9"></a>./bootstrap.sh
<a id="__codelineno-75-10" name="__codelineno-75-10" href="#__codelineno-75-10"></a>
<a id="__codelineno-75-11" name="__codelineno-75-11" href="#__codelineno-75-11"></a><span class="c1"># 4. Wait for ArgoCD to sync all applications</span>
<a id="__codelineno-75-12" name="__codelineno-75-12" href="#__codelineno-75-12"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-w
<a id="__codelineno-75-13" name="__codelineno-75-13" href="#__codelineno-75-13"></a>
<a id="__codelineno-75-14" name="__codelineno-75-14" href="#__codelineno-75-14"></a><span class="c1"># 5. Recreate any unsealed secrets (from password manager)</span>
<a id="__codelineno-75-15" name="__codelineno-75-15" href="#__codelineno-75-15"></a><span class="c1"># 6. Configure DNS for new cluster IPs</span>
<a id="__codelineno-75-16" name="__codelineno-75-16" href="#__codelineno-75-16"></a><span class="c1"># 7. Verify all applications are healthy</span>
</code></pre></div>
<p><strong>Time Estimate</strong>: 30-60 minutes</p>
<p><strong>Data Loss</strong>:
- Ephemeral data: Lost
- Database data: Lost (no backups currently)
- Configuration: No loss (in Git)</p>
<h3 id="future-backup-plan">Future Backup Plan<a class="headerlink" href="#future-backup-plan" title="Permanent link">&para;</a></h3>
<p><strong>Recommended</strong>:</p>
<ol>
<li>
<p><strong>Velero</strong> for cluster backups
<div class="highlight"><pre><span></span><code><a id="__codelineno-76-1" name="__codelineno-76-1" href="#__codelineno-76-1"></a>helm<span class="w"> </span>install<span class="w"> </span>velero<span class="w"> </span>vmware-tanzu/velero<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-76-2" name="__codelineno-76-2" href="#__codelineno-76-2"></a><span class="w"> </span>--namespace<span class="w"> </span>velero<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-76-3" name="__codelineno-76-3" href="#__codelineno-76-3"></a><span class="w"> </span>--create-namespace<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-76-4" name="__codelineno-76-4" href="#__codelineno-76-4"></a><span class="w"> </span>--set<span class="w"> </span>configuration.provider<span class="o">=</span>aws<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-76-5" name="__codelineno-76-5" href="#__codelineno-76-5"></a><span class="w"> </span>--set<span class="w"> </span>configuration.backupStorageLocation<span class="o">[</span><span class="m">0</span><span class="o">]</span>.bucket<span class="o">=</span>cluster-backups
</code></pre></div></p>
</li>
<li>
<p><strong>PostgreSQL backups</strong> via CronJob
<div class="highlight"><pre><span></span><code><a id="__codelineno-77-1" name="__codelineno-77-1" href="#__codelineno-77-1"></a><span class="c1"># pg-backup-cronjob.yaml</span>
<a id="__codelineno-77-2" name="__codelineno-77-2" href="#__codelineno-77-2"></a><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">CronJob</span>
<a id="__codelineno-77-3" name="__codelineno-77-3" href="#__codelineno-77-3"></a><span class="nt">spec</span><span class="p">:</span>
<a id="__codelineno-77-4" name="__codelineno-77-4" href="#__codelineno-77-4"></a><span class="w"> </span><span class="nt">schedule</span><span class="p">:</span><span class="w"> </span><span class="s">&quot;0</span><span class="nv"> </span><span class="s">2</span><span class="nv"> </span><span class="s">*</span><span class="nv"> </span><span class="s">*</span><span class="nv"> </span><span class="s">*&quot;</span><span class="w"> </span><span class="c1"># Daily at 2am</span>
<a id="__codelineno-77-5" name="__codelineno-77-5" href="#__codelineno-77-5"></a><span class="w"> </span><span class="nt">jobTemplate</span><span class="p">:</span>
<a id="__codelineno-77-6" name="__codelineno-77-6" href="#__codelineno-77-6"></a><span class="w"> </span><span class="nt">spec</span><span class="p">:</span>
<a id="__codelineno-77-7" name="__codelineno-77-7" href="#__codelineno-77-7"></a><span class="w"> </span><span class="nt">template</span><span class="p">:</span>
<a id="__codelineno-77-8" name="__codelineno-77-8" href="#__codelineno-77-8"></a><span class="w"> </span><span class="nt">spec</span><span class="p">:</span>
<a id="__codelineno-77-9" name="__codelineno-77-9" href="#__codelineno-77-9"></a><span class="w"> </span><span class="nt">containers</span><span class="p">:</span>
<a id="__codelineno-77-10" name="__codelineno-77-10" href="#__codelineno-77-10"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">pg-dump</span>
<a id="__codelineno-77-11" name="__codelineno-77-11" href="#__codelineno-77-11"></a><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">postgres:16-alpine</span>
<a id="__codelineno-77-12" name="__codelineno-77-12" href="#__codelineno-77-12"></a><span class="w"> </span><span class="nt">command</span><span class="p">:</span>
<a id="__codelineno-77-13" name="__codelineno-77-13" href="#__codelineno-77-13"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">/bin/sh</span>
<a id="__codelineno-77-14" name="__codelineno-77-14" href="#__codelineno-77-14"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">-c</span>
<a id="__codelineno-77-15" name="__codelineno-77-15" href="#__codelineno-77-15"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="l l-Scalar l-Scalar-Plain">pg_dump -U $DB_USER -d $DB_NAME &gt; /backup/dump-$(date +%Y%m%d).sql</span>
</code></pre></div></p>
</li>
<li>
<p><strong>Sealed Secrets private key backup</strong>
<div class="highlight"><pre><span></span><code><a id="__codelineno-78-1" name="__codelineno-78-1" href="#__codelineno-78-1"></a><span class="c1"># Backup sealed-secrets controller private key</span>
<a id="__codelineno-78-2" name="__codelineno-78-2" href="#__codelineno-78-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>secret<span class="w"> </span>-n<span class="w"> </span>kube-system<span class="w"> </span>sealed-secrets-key<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-78-3" name="__codelineno-78-3" href="#__codelineno-78-3"></a><span class="w"> </span>-o<span class="w"> </span>yaml<span class="w"> </span>&gt;<span class="w"> </span>sealed-secrets-key-backup.yaml
<a id="__codelineno-78-4" name="__codelineno-78-4" href="#__codelineno-78-4"></a>
<a id="__codelineno-78-5" name="__codelineno-78-5" href="#__codelineno-78-5"></a><span class="c1"># Store in secure location (password manager, vault)</span>
</code></pre></div></p>
</li>
</ol>
<hr />
<h2 id="maintenance-procedures">Maintenance Procedures<a class="headerlink" href="#maintenance-procedures" title="Permanent link">&para;</a></h2>
<h3 id="upgrading-argocd">Upgrading ArgoCD<a class="headerlink" href="#upgrading-argocd" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-79-1" name="__codelineno-79-1" href="#__codelineno-79-1"></a><span class="c1"># Check current version</span>
<a id="__codelineno-79-2" name="__codelineno-79-2" href="#__codelineno-79-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>deployment<span class="w"> </span>argocd-server<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-79-3" name="__codelineno-79-3" href="#__codelineno-79-3"></a><span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.spec.template.spec.containers[0].image}&#39;</span>
<a id="__codelineno-79-4" name="__codelineno-79-4" href="#__codelineno-79-4"></a>
<a id="__codelineno-79-5" name="__codelineno-79-5" href="#__codelineno-79-5"></a><span class="c1"># Update version in values</span>
<a id="__codelineno-79-6" name="__codelineno-79-6" href="#__codelineno-79-6"></a>vim<span class="w"> </span>infra/values/base/argocd-values.yaml
<a id="__codelineno-79-7" name="__codelineno-79-7" href="#__codelineno-79-7"></a>
<a id="__codelineno-79-8" name="__codelineno-79-8" href="#__codelineno-79-8"></a><span class="c1"># Or upgrade via Helm directly</span>
<a id="__codelineno-79-9" name="__codelineno-79-9" href="#__codelineno-79-9"></a>helm<span class="w"> </span>upgrade<span class="w"> </span>argocd<span class="w"> </span>argo-cd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-79-10" name="__codelineno-79-10" href="#__codelineno-79-10"></a><span class="w"> </span>--repo<span class="w"> </span>https://argoproj.github.io/argo-helm<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-79-11" name="__codelineno-79-11" href="#__codelineno-79-11"></a><span class="w"> </span>--namespace<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-79-12" name="__codelineno-79-12" href="#__codelineno-79-12"></a><span class="w"> </span>--values<span class="w"> </span>infra/values/base/argocd-values.yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-79-13" name="__codelineno-79-13" href="#__codelineno-79-13"></a><span class="w"> </span>--version<span class="w"> </span><span class="m">6</span>.0.0<span class="w"> </span><span class="c1"># New version</span>
<a id="__codelineno-79-14" name="__codelineno-79-14" href="#__codelineno-79-14"></a>
<a id="__codelineno-79-15" name="__codelineno-79-15" href="#__codelineno-79-15"></a><span class="c1"># Verify</span>
<a id="__codelineno-79-16" name="__codelineno-79-16" href="#__codelineno-79-16"></a>kubectl<span class="w"> </span>get<span class="w"> </span>pods<span class="w"> </span>-n<span class="w"> </span>argocd
</code></pre></div>
<h3 id="upgrading-kubernetes-version">Upgrading Kubernetes Version<a class="headerlink" href="#upgrading-kubernetes-version" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-80-1" name="__codelineno-80-1" href="#__codelineno-80-1"></a><span class="c1"># UpCloud: Upgrade via control panel or CLI</span>
<a id="__codelineno-80-2" name="__codelineno-80-2" href="#__codelineno-80-2"></a>
<a id="__codelineno-80-3" name="__codelineno-80-3" href="#__codelineno-80-3"></a><span class="c1"># After upgrade, verify cluster</span>
<a id="__codelineno-80-4" name="__codelineno-80-4" href="#__codelineno-80-4"></a>kubectl<span class="w"> </span>version
<a id="__codelineno-80-5" name="__codelineno-80-5" href="#__codelineno-80-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>nodes
<a id="__codelineno-80-6" name="__codelineno-80-6" href="#__codelineno-80-6"></a>
<a id="__codelineno-80-7" name="__codelineno-80-7" href="#__codelineno-80-7"></a><span class="c1"># Check for deprecated APIs</span>
<a id="__codelineno-80-8" name="__codelineno-80-8" href="#__codelineno-80-8"></a>kubectl<span class="w"> </span>api-resources
<a id="__codelineno-80-9" name="__codelineno-80-9" href="#__codelineno-80-9"></a>
<a id="__codelineno-80-10" name="__codelineno-80-10" href="#__codelineno-80-10"></a><span class="c1"># Update any deprecated resources in Git</span>
</code></pre></div>
<h3 id="rotating-tls-certificates">Rotating TLS Certificates<a class="headerlink" href="#rotating-tls-certificates" title="Permanent link">&para;</a></h3>
<p>Let's Encrypt certificates auto-renew, but if manual rotation is needed:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-81-1" name="__codelineno-81-1" href="#__codelineno-81-1"></a><span class="c1"># Delete certificate to force renewal</span>
<a id="__codelineno-81-2" name="__codelineno-81-2" href="#__codelineno-81-2"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>certificate<span class="w"> </span>myapp-tls<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-81-3" name="__codelineno-81-3" href="#__codelineno-81-3"></a>
<a id="__codelineno-81-4" name="__codelineno-81-4" href="#__codelineno-81-4"></a><span class="c1"># Cert-manager will automatically recreate</span>
<a id="__codelineno-81-5" name="__codelineno-81-5" href="#__codelineno-81-5"></a>kubectl<span class="w"> </span>get<span class="w"> </span>certificate<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>-w
</code></pre></div>
<h3 id="cleaning-up-old-resources">Cleaning Up Old Resources<a class="headerlink" href="#cleaning-up-old-resources" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-82-1" name="__codelineno-82-1" href="#__codelineno-82-1"></a><span class="c1"># List all namespaces</span>
<a id="__codelineno-82-2" name="__codelineno-82-2" href="#__codelineno-82-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>namespaces
<a id="__codelineno-82-3" name="__codelineno-82-3" href="#__codelineno-82-3"></a>
<a id="__codelineno-82-4" name="__codelineno-82-4" href="#__codelineno-82-4"></a><span class="c1"># Remove unused namespaces</span>
<a id="__codelineno-82-5" name="__codelineno-82-5" href="#__codelineno-82-5"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>namespace<span class="w"> </span>old-app
<a id="__codelineno-82-6" name="__codelineno-82-6" href="#__codelineno-82-6"></a>
<a id="__codelineno-82-7" name="__codelineno-82-7" href="#__codelineno-82-7"></a><span class="c1"># Clean up ArgoCD applications</span>
<a id="__codelineno-82-8" name="__codelineno-82-8" href="#__codelineno-82-8"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-82-9" name="__codelineno-82-9" href="#__codelineno-82-9"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>application<span class="w"> </span>old-app<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-82-10" name="__codelineno-82-10" href="#__codelineno-82-10"></a>
<a id="__codelineno-82-11" name="__codelineno-82-11" href="#__codelineno-82-11"></a><span class="c1"># Clean up old Docker images (on nodes)</span>
<a id="__codelineno-82-12" name="__codelineno-82-12" href="#__codelineno-82-12"></a><span class="c1"># SSH to nodes and run:</span>
<a id="__codelineno-82-13" name="__codelineno-82-13" href="#__codelineno-82-13"></a>docker<span class="w"> </span>image<span class="w"> </span>prune<span class="w"> </span>-a<span class="w"> </span>--filter<span class="w"> </span><span class="s2">&quot;until=720h&quot;</span><span class="w"> </span><span class="c1"># 30 days</span>
</code></pre></div>
<h3 id="dns-management">DNS Management<a class="headerlink" href="#dns-management" title="Permanent link">&para;</a></h3>
<p><strong>Adding New Subdomain</strong>:</p>
<ol>
<li>
<p>Add DNS A record pointing to Traefik LoadBalancer IP
<div class="highlight"><pre><span></span><code><a id="__codelineno-83-1" name="__codelineno-83-1" href="#__codelineno-83-1"></a><span class="c1"># Get LoadBalancer IP</span>
<a id="__codelineno-83-2" name="__codelineno-83-2" href="#__codelineno-83-2"></a>kubectl<span class="w"> </span>get<span class="w"> </span>svc<span class="w"> </span>-n<span class="w"> </span>traefik<span class="w"> </span>traefik<span class="w"> </span>-o<span class="w"> </span><span class="nv">jsonpath</span><span class="o">=</span><span class="s1">&#39;{.status.loadBalancer.ingress[0].ip}&#39;</span>
</code></pre></div></p>
</li>
<li>
<p>Add to DNS provider:
<div class="highlight"><pre><span></span><code><a id="__codelineno-84-1" name="__codelineno-84-1" href="#__codelineno-84-1"></a>myapp.forteapps.net A &lt;LoadBalancer-IP&gt;
</code></pre></div></p>
</li>
<li>
<p>Verify DNS propagation:
<div class="highlight"><pre><span></span><code><a id="__codelineno-85-1" name="__codelineno-85-1" href="#__codelineno-85-1"></a>nslookup<span class="w"> </span>myapp.forteapps.net
<a id="__codelineno-85-2" name="__codelineno-85-2" href="#__codelineno-85-2"></a>dig<span class="w"> </span>myapp.forteapps.net
</code></pre></div></p>
</li>
</ol>
<h3 id="monitoring-resource-usage">Monitoring Resource Usage<a class="headerlink" href="#monitoring-resource-usage" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-86-1" name="__codelineno-86-1" href="#__codelineno-86-1"></a><span class="c1"># Node resource usage</span>
<a id="__codelineno-86-2" name="__codelineno-86-2" href="#__codelineno-86-2"></a>kubectl<span class="w"> </span>top<span class="w"> </span>nodes
<a id="__codelineno-86-3" name="__codelineno-86-3" href="#__codelineno-86-3"></a>
<a id="__codelineno-86-4" name="__codelineno-86-4" href="#__codelineno-86-4"></a><span class="c1"># Pod resource usage</span>
<a id="__codelineno-86-5" name="__codelineno-86-5" href="#__codelineno-86-5"></a>kubectl<span class="w"> </span>top<span class="w"> </span>pods<span class="w"> </span>--all-namespaces
<a id="__codelineno-86-6" name="__codelineno-86-6" href="#__codelineno-86-6"></a>
<a id="__codelineno-86-7" name="__codelineno-86-7" href="#__codelineno-86-7"></a><span class="c1"># Identify resource hogs</span>
<a id="__codelineno-86-8" name="__codelineno-86-8" href="#__codelineno-86-8"></a>kubectl<span class="w"> </span>top<span class="w"> </span>pods<span class="w"> </span>--all-namespaces<span class="w"> </span>--sort-by<span class="o">=</span>memory
<a id="__codelineno-86-9" name="__codelineno-86-9" href="#__codelineno-86-9"></a>kubectl<span class="w"> </span>top<span class="w"> </span>pods<span class="w"> </span>--all-namespaces<span class="w"> </span>--sort-by<span class="o">=</span>cpu
</code></pre></div>
<hr />
<h2 id="advanced-operations">Advanced Operations<a class="headerlink" href="#advanced-operations" title="Permanent link">&para;</a></h2>
<h3 id="adding-a-new-infrastructure-component">Adding a New Infrastructure Component<a class="headerlink" href="#adding-a-new-infrastructure-component" title="Permanent link">&para;</a></h3>
<p>Example: Adding Redis</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-87-1" name="__codelineno-87-1" href="#__codelineno-87-1"></a><span class="c1"># 1. Create application manifest in base/</span>
<a id="__codelineno-87-2" name="__codelineno-87-2" href="#__codelineno-87-2"></a>cat<span class="w"> </span>&gt;<span class="w"> </span>infra/base/redis-application.yaml<span class="w"> </span><span class="s">&lt;&lt;EOF</span>
<a id="__codelineno-87-3" name="__codelineno-87-3" href="#__codelineno-87-3"></a><span class="s">apiVersion: argoproj.io/v1alpha1</span>
<a id="__codelineno-87-4" name="__codelineno-87-4" href="#__codelineno-87-4"></a><span class="s">kind: Application</span>
<a id="__codelineno-87-5" name="__codelineno-87-5" href="#__codelineno-87-5"></a><span class="s">metadata:</span>
<a id="__codelineno-87-6" name="__codelineno-87-6" href="#__codelineno-87-6"></a><span class="s"> name: redis</span>
<a id="__codelineno-87-7" name="__codelineno-87-7" href="#__codelineno-87-7"></a><span class="s"> namespace: argocd</span>
<a id="__codelineno-87-8" name="__codelineno-87-8" href="#__codelineno-87-8"></a><span class="s"> annotations:</span>
<a id="__codelineno-87-9" name="__codelineno-87-9" href="#__codelineno-87-9"></a><span class="s"> argocd.argoproj.io/sync-wave: &quot;1&quot;</span>
<a id="__codelineno-87-10" name="__codelineno-87-10" href="#__codelineno-87-10"></a><span class="s">spec:</span>
<a id="__codelineno-87-11" name="__codelineno-87-11" href="#__codelineno-87-11"></a><span class="s"> project: default</span>
<a id="__codelineno-87-12" name="__codelineno-87-12" href="#__codelineno-87-12"></a><span class="s"> sources:</span>
<a id="__codelineno-87-13" name="__codelineno-87-13" href="#__codelineno-87-13"></a><span class="s"> - repoURL: https://charts.bitnami.com/bitnami</span>
<a id="__codelineno-87-14" name="__codelineno-87-14" href="#__codelineno-87-14"></a><span class="s"> chart: redis</span>
<a id="__codelineno-87-15" name="__codelineno-87-15" href="#__codelineno-87-15"></a><span class="s"> targetRevision: 18.0.0</span>
<a id="__codelineno-87-16" name="__codelineno-87-16" href="#__codelineno-87-16"></a><span class="s"> helm:</span>
<a id="__codelineno-87-17" name="__codelineno-87-17" href="#__codelineno-87-17"></a><span class="s"> releaseName: redis</span>
<a id="__codelineno-87-18" name="__codelineno-87-18" href="#__codelineno-87-18"></a><span class="s"> valueFiles:</span>
<a id="__codelineno-87-19" name="__codelineno-87-19" href="#__codelineno-87-19"></a><span class="s"> - \$values/infra/values/base/redis-values.yaml</span>
<a id="__codelineno-87-20" name="__codelineno-87-20" href="#__codelineno-87-20"></a><span class="s"> - repoURL: ssh://git@git.forteapps.net:2222/Forte/launchpad.git</span>
<a id="__codelineno-87-21" name="__codelineno-87-21" href="#__codelineno-87-21"></a><span class="s"> targetRevision: HEAD</span>
<a id="__codelineno-87-22" name="__codelineno-87-22" href="#__codelineno-87-22"></a><span class="s"> ref: values</span>
<a id="__codelineno-87-23" name="__codelineno-87-23" href="#__codelineno-87-23"></a><span class="s"> destination:</span>
<a id="__codelineno-87-24" name="__codelineno-87-24" href="#__codelineno-87-24"></a><span class="s"> server: https://kubernetes.default.svc</span>
<a id="__codelineno-87-25" name="__codelineno-87-25" href="#__codelineno-87-25"></a><span class="s"> namespace: redis</span>
<a id="__codelineno-87-26" name="__codelineno-87-26" href="#__codelineno-87-26"></a><span class="s"> syncPolicy:</span>
<a id="__codelineno-87-27" name="__codelineno-87-27" href="#__codelineno-87-27"></a><span class="s"> automated:</span>
<a id="__codelineno-87-28" name="__codelineno-87-28" href="#__codelineno-87-28"></a><span class="s"> prune: true</span>
<a id="__codelineno-87-29" name="__codelineno-87-29" href="#__codelineno-87-29"></a><span class="s"> selfHeal: true</span>
<a id="__codelineno-87-30" name="__codelineno-87-30" href="#__codelineno-87-30"></a><span class="s"> syncOptions:</span>
<a id="__codelineno-87-31" name="__codelineno-87-31" href="#__codelineno-87-31"></a><span class="s"> - CreateNamespace=true</span>
<a id="__codelineno-87-32" name="__codelineno-87-32" href="#__codelineno-87-32"></a><span class="s">EOF</span>
<a id="__codelineno-87-33" name="__codelineno-87-33" href="#__codelineno-87-33"></a>
<a id="__codelineno-87-34" name="__codelineno-87-34" href="#__codelineno-87-34"></a><span class="c1"># 2. Add to base kustomization</span>
<a id="__codelineno-87-35" name="__codelineno-87-35" href="#__codelineno-87-35"></a><span class="c1"># Edit infra/base/kustomization.yaml and add: - redis-application.yaml</span>
<a id="__codelineno-87-36" name="__codelineno-87-36" href="#__codelineno-87-36"></a>
<a id="__codelineno-87-37" name="__codelineno-87-37" href="#__codelineno-87-37"></a><span class="c1"># 3. Create base values file</span>
<a id="__codelineno-87-38" name="__codelineno-87-38" href="#__codelineno-87-38"></a>cat<span class="w"> </span>&gt;<span class="w"> </span>infra/values/base/redis-values.yaml<span class="w"> </span><span class="s">&lt;&lt;EOF</span>
<a id="__codelineno-87-39" name="__codelineno-87-39" href="#__codelineno-87-39"></a><span class="s">auth:</span>
<a id="__codelineno-87-40" name="__codelineno-87-40" href="#__codelineno-87-40"></a><span class="s"> enabled: true</span>
<a id="__codelineno-87-41" name="__codelineno-87-41" href="#__codelineno-87-41"></a><span class="s">EOF</span>
<a id="__codelineno-87-42" name="__codelineno-87-42" href="#__codelineno-87-42"></a>
<a id="__codelineno-87-43" name="__codelineno-87-43" href="#__codelineno-87-43"></a><span class="c1"># 4. Commit and push</span>
<a id="__codelineno-87-44" name="__codelineno-87-44" href="#__codelineno-87-44"></a>git<span class="w"> </span>add<span class="w"> </span>infra/base/redis-application.yaml<span class="w"> </span>infra/values/base/redis-values.yaml<span class="w"> </span>infra/base/kustomization.yaml
<a id="__codelineno-87-45" name="__codelineno-87-45" href="#__codelineno-87-45"></a>git<span class="w"> </span>commit<span class="w"> </span>-m<span class="w"> </span><span class="s2">&quot;Add Redis infrastructure component&quot;</span>
<a id="__codelineno-87-46" name="__codelineno-87-46" href="#__codelineno-87-46"></a>git<span class="w"> </span>push
<a id="__codelineno-87-47" name="__codelineno-87-47" href="#__codelineno-87-47"></a>
<a id="__codelineno-87-48" name="__codelineno-87-48" href="#__codelineno-87-48"></a><span class="c1"># 5. ArgoCD will auto-sync within 60 seconds</span>
</code></pre></div>
<h3 id="multi-cluster-setup">Multi-Cluster Setup<a class="headerlink" href="#multi-cluster-setup" title="Permanent link">&para;</a></h3>
<p>The repository supports multiple clusters via Kustomize overlays:</p>
<ul>
<li><strong>upc-dev</strong> (default): <code>infra/overlays/upc-dev/</code> — uses base Applications as-is</li>
<li><strong>upc-prod</strong>: <code>infra/overlays/upc-prod/</code> — patches value file paths from <code>upc-dev</code> to <code>upc-prod</code></li>
</ul>
<p>Each cluster has its own:
- Root app-of-apps file: <code>_app-of-apps-upc-dev.yaml</code> / <code>_app-of-apps-upc-prod.yaml</code>
- Cluster-specific Helm values: <code>infra/values/upc-dev/</code> / <code>infra/values/upc-prod/</code>
- Sealed secrets: <code>secrets/upc-dev/</code> (others as needed)
- Apps overlay: <code>apps/overlays/upc-dev/</code> / <code>apps/overlays/upc-prod/</code></p>
<p>To add a new cluster, create a new overlay directory (e.g., <code>infra/overlays/upc-staging/</code>) with patches that swap the value file paths.</p>
<h3 id="blue-green-deployments">Blue-Green Deployments<a class="headerlink" href="#blue-green-deployments" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-88-1" name="__codelineno-88-1" href="#__codelineno-88-1"></a><span class="c1"># Deploy blue version</span>
<a id="__codelineno-88-2" name="__codelineno-88-2" href="#__codelineno-88-2"></a>helm<span class="w"> </span>install<span class="w"> </span>myapp-blue<span class="w"> </span>forteapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-88-3" name="__codelineno-88-3" href="#__codelineno-88-3"></a><span class="w"> </span>--set<span class="w"> </span>app.image.tag<span class="o">=</span>v1.0.0
<a id="__codelineno-88-4" name="__codelineno-88-4" href="#__codelineno-88-4"></a>
<a id="__codelineno-88-5" name="__codelineno-88-5" href="#__codelineno-88-5"></a><span class="c1"># Deploy green version</span>
<a id="__codelineno-88-6" name="__codelineno-88-6" href="#__codelineno-88-6"></a>helm<span class="w"> </span>install<span class="w"> </span>myapp-green<span class="w"> </span>forteapp<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-88-7" name="__codelineno-88-7" href="#__codelineno-88-7"></a><span class="w"> </span>--set<span class="w"> </span>app.image.tag<span class="o">=</span>v2.0.0
<a id="__codelineno-88-8" name="__codelineno-88-8" href="#__codelineno-88-8"></a>
<a id="__codelineno-88-9" name="__codelineno-88-9" href="#__codelineno-88-9"></a><span class="c1"># Switch traffic via IngressRoute</span>
<a id="__codelineno-88-10" name="__codelineno-88-10" href="#__codelineno-88-10"></a>kubectl<span class="w"> </span>patch<span class="w"> </span>ingressroute<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>--type<span class="w"> </span>merge<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-88-11" name="__codelineno-88-11" href="#__codelineno-88-11"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;spec&quot;:{&quot;routes&quot;:[{&quot;services&quot;:[{&quot;name&quot;:&quot;myapp-green&quot;}]}]}}&#39;</span>
<a id="__codelineno-88-12" name="__codelineno-88-12" href="#__codelineno-88-12"></a>
<a id="__codelineno-88-13" name="__codelineno-88-13" href="#__codelineno-88-13"></a><span class="c1"># Remove blue deployment after validation</span>
<a id="__codelineno-88-14" name="__codelineno-88-14" href="#__codelineno-88-14"></a>helm<span class="w"> </span>uninstall<span class="w"> </span>myapp-blue
</code></pre></div>
<hr />
<h2 id="emergency-procedures">Emergency Procedures<a class="headerlink" href="#emergency-procedures" title="Permanent link">&para;</a></h2>
<h3 id="emergency-rollback">Emergency Rollback<a class="headerlink" href="#emergency-rollback" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-89-1" name="__codelineno-89-1" href="#__codelineno-89-1"></a><span class="c1"># Immediate rollback</span>
<a id="__codelineno-89-2" name="__codelineno-89-2" href="#__codelineno-89-2"></a>kubectl<span class="w"> </span>rollout<span class="w"> </span>undo<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp
<a id="__codelineno-89-3" name="__codelineno-89-3" href="#__codelineno-89-3"></a>
<a id="__codelineno-89-4" name="__codelineno-89-4" href="#__codelineno-89-4"></a><span class="c1"># Update Git to make permanent</span>
<a id="__codelineno-89-5" name="__codelineno-89-5" href="#__codelineno-89-5"></a><span class="nb">cd</span><span class="w"> </span>~/dev/k8s/helm-prod-values
<a id="__codelineno-89-6" name="__codelineno-89-6" href="#__codelineno-89-6"></a>git<span class="w"> </span>revert<span class="w"> </span>HEAD
<a id="__codelineno-89-7" name="__codelineno-89-7" href="#__codelineno-89-7"></a>git<span class="w"> </span>push
</code></pre></div>
<h3 id="emergency-scale-down">Emergency Scale Down<a class="headerlink" href="#emergency-scale-down" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-90-1" name="__codelineno-90-1" href="#__codelineno-90-1"></a><span class="c1"># Scale to zero (maintenance mode)</span>
<a id="__codelineno-90-2" name="__codelineno-90-2" href="#__codelineno-90-2"></a>kubectl<span class="w"> </span>scale<span class="w"> </span>deployment<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>myapp<span class="w"> </span>--replicas<span class="o">=</span><span class="m">0</span>
<a id="__codelineno-90-3" name="__codelineno-90-3" href="#__codelineno-90-3"></a>
<a id="__codelineno-90-4" name="__codelineno-90-4" href="#__codelineno-90-4"></a><span class="c1"># Update Git</span>
<a id="__codelineno-90-5" name="__codelineno-90-5" href="#__codelineno-90-5"></a>vim<span class="w"> </span>helm-values/myapp/values.yaml
<a id="__codelineno-90-6" name="__codelineno-90-6" href="#__codelineno-90-6"></a><span class="c1"># Set replicaCount: 0</span>
<a id="__codelineno-90-7" name="__codelineno-90-7" href="#__codelineno-90-7"></a>git<span class="w"> </span>commit<span class="w"> </span>-am<span class="w"> </span><span class="s2">&quot;Scale down myapp for maintenance&quot;</span>
<a id="__codelineno-90-8" name="__codelineno-90-8" href="#__codelineno-90-8"></a>git<span class="w"> </span>push
</code></pre></div>
<h3 id="emergency-application-removal">Emergency Application Removal<a class="headerlink" href="#emergency-application-removal" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-91-1" name="__codelineno-91-1" href="#__codelineno-91-1"></a><span class="c1"># Remove application but keep data</span>
<a id="__codelineno-91-2" name="__codelineno-91-2" href="#__codelineno-91-2"></a>kubectl<span class="w"> </span>patch<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-91-3" name="__codelineno-91-3" href="#__codelineno-91-3"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;metadata&quot;:{&quot;finalizers&quot;:[]}}&#39;</span><span class="w"> </span>--type<span class="w"> </span>merge
<a id="__codelineno-91-4" name="__codelineno-91-4" href="#__codelineno-91-4"></a>kubectl<span class="w"> </span>delete<span class="w"> </span>application<span class="w"> </span>myapp<span class="w"> </span>-n<span class="w"> </span>argocd
<a id="__codelineno-91-5" name="__codelineno-91-5" href="#__codelineno-91-5"></a>
<a id="__codelineno-91-6" name="__codelineno-91-6" href="#__codelineno-91-6"></a><span class="c1"># Resources remain in cluster</span>
</code></pre></div>
<hr />
<h2 id="useful-scripts">Useful Scripts<a class="headerlink" href="#useful-scripts" title="Permanent link">&para;</a></h2>
<h3 id="sync-all-applications">Sync All Applications<a class="headerlink" href="#sync-all-applications" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-92-1" name="__codelineno-92-1" href="#__codelineno-92-1"></a><span class="ch">#!/bin/bash</span>
<a id="__codelineno-92-2" name="__codelineno-92-2" href="#__codelineno-92-2"></a><span class="c1"># sync-all.sh</span>
<a id="__codelineno-92-3" name="__codelineno-92-3" href="#__codelineno-92-3"></a><span class="k">for</span><span class="w"> </span>app<span class="w"> </span><span class="k">in</span><span class="w"> </span><span class="k">$(</span>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span>-o<span class="w"> </span>name<span class="k">)</span><span class="p">;</span><span class="w"> </span><span class="k">do</span>
<a id="__codelineno-92-4" name="__codelineno-92-4" href="#__codelineno-92-4"></a><span class="w"> </span>kubectl<span class="w"> </span>patch<span class="w"> </span><span class="nv">$app</span><span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-92-5" name="__codelineno-92-5" href="#__codelineno-92-5"></a><span class="w"> </span>--type<span class="w"> </span>merge<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-92-6" name="__codelineno-92-6" href="#__codelineno-92-6"></a><span class="w"> </span>-p<span class="w"> </span><span class="s1">&#39;{&quot;metadata&quot;:{&quot;annotations&quot;:{&quot;argocd.argoproj.io/refresh&quot;:&quot;hard&quot;}}}&#39;</span>
<a id="__codelineno-92-7" name="__codelineno-92-7" href="#__codelineno-92-7"></a><span class="k">done</span>
</code></pre></div>
<h3 id="check-all-applications-health">Check All Applications Health<a class="headerlink" href="#check-all-applications-health" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-93-1" name="__codelineno-93-1" href="#__codelineno-93-1"></a><span class="ch">#!/bin/bash</span>
<a id="__codelineno-93-2" name="__codelineno-93-2" href="#__codelineno-93-2"></a><span class="c1"># health-check.sh</span>
<a id="__codelineno-93-3" name="__codelineno-93-3" href="#__codelineno-93-3"></a>kubectl<span class="w"> </span>get<span class="w"> </span>applications<span class="w"> </span>-n<span class="w"> </span>argocd<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-93-4" name="__codelineno-93-4" href="#__codelineno-93-4"></a><span class="w"> </span>-o<span class="w"> </span>custom-columns<span class="o">=</span><span class="se">\</span>
<a id="__codelineno-93-5" name="__codelineno-93-5" href="#__codelineno-93-5"></a>NAME:.metadata.name,<span class="se">\</span>
<a id="__codelineno-93-6" name="__codelineno-93-6" href="#__codelineno-93-6"></a>SYNC:.status.sync.status,<span class="se">\</span>
<a id="__codelineno-93-7" name="__codelineno-93-7" href="#__codelineno-93-7"></a>HEALTH:.status.health.status,<span class="se">\</span>
<a id="__codelineno-93-8" name="__codelineno-93-8" href="#__codelineno-93-8"></a>MESSAGE:.status.health.message
</code></pre></div>
<h3 id="seal-secret-helper">Seal Secret Helper<a class="headerlink" href="#seal-secret-helper" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-94-1" name="__codelineno-94-1" href="#__codelineno-94-1"></a><span class="ch">#!/bin/bash</span>
<a id="__codelineno-94-2" name="__codelineno-94-2" href="#__codelineno-94-2"></a><span class="c1"># seal-secret.sh</span>
<a id="__codelineno-94-3" name="__codelineno-94-3" href="#__codelineno-94-3"></a><span class="nv">NAMESPACE</span><span class="o">=</span><span class="si">${</span><span class="nv">1</span><span class="k">:-</span><span class="nv">default</span><span class="si">}</span>
<a id="__codelineno-94-4" name="__codelineno-94-4" href="#__codelineno-94-4"></a><span class="nv">SECRET_FILE</span><span class="o">=</span><span class="si">${</span><span class="nv">2</span><span class="k">:-</span><span class="nv">private</span><span class="p">/secret.yaml</span><span class="si">}</span>
<a id="__codelineno-94-5" name="__codelineno-94-5" href="#__codelineno-94-5"></a><span class="nv">OUTPUT_FILE</span><span class="o">=</span><span class="si">${</span><span class="nv">3</span><span class="k">:-</span><span class="nv">secrets</span><span class="p">/secret-sealed.yaml</span><span class="si">}</span>
<a id="__codelineno-94-6" name="__codelineno-94-6" href="#__codelineno-94-6"></a>
<a id="__codelineno-94-7" name="__codelineno-94-7" href="#__codelineno-94-7"></a>kubeseal<span class="w"> </span>--format<span class="o">=</span>yaml<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-94-8" name="__codelineno-94-8" href="#__codelineno-94-8"></a><span class="w"> </span>--cert<span class="o">=</span>pub-cert.pem<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-94-9" name="__codelineno-94-9" href="#__codelineno-94-9"></a><span class="w"> </span>--namespace<span class="o">=</span><span class="nv">$NAMESPACE</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-94-10" name="__codelineno-94-10" href="#__codelineno-94-10"></a><span class="w"> </span>&lt;<span class="w"> </span><span class="nv">$SECRET_FILE</span><span class="w"> </span><span class="se">\</span>
<a id="__codelineno-94-11" name="__codelineno-94-11" href="#__codelineno-94-11"></a><span class="w"> </span>&gt;<span class="w"> </span><span class="nv">$OUTPUT_FILE</span>
<a id="__codelineno-94-12" name="__codelineno-94-12" href="#__codelineno-94-12"></a>
<a id="__codelineno-94-13" name="__codelineno-94-13" href="#__codelineno-94-13"></a><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Sealed secret created: </span><span class="nv">$OUTPUT_FILE</span><span class="s2">&quot;</span>
<a id="__codelineno-94-14" name="__codelineno-94-14" href="#__codelineno-94-14"></a><span class="nb">echo</span><span class="w"> </span><span class="s2">&quot;Remember to delete: </span><span class="nv">$SECRET_FILE</span><span class="s2">&quot;</span>
</code></pre></div>
<hr />
<h2 id="checklist-templates">Checklist Templates<a class="headerlink" href="#checklist-templates" title="Permanent link">&para;</a></h2>
<h3 id="new-application-deployment-checklist">New Application Deployment Checklist<a class="headerlink" href="#new-application-deployment-checklist" title="Permanent link">&para;</a></h3>
<ul>
<li>[ ] Application code repository created</li>
<li>[ ] Dockerfile created and tested</li>
<li>[ ] GitHub Actions workflow configured</li>
<li>[ ] Helm values created in <code>helm-prod-values/</code></li>
<li>[ ] ArgoCD application manifest created in <code>apps/</code></li>
<li>[ ] Secrets created and sealed</li>
<li>[ ] DNS record added for domain</li>
<li>[ ] Application synced successfully</li>
<li>[ ] Health check passed</li>
<li>[ ] Slack notification received</li>
<li>[ ] Application accessible via domain</li>
<li>[ ] Monitoring configured</li>
<li>[ ] Documentation updated</li>
</ul>
<h3 id="incident-response-checklist">Incident Response Checklist<a class="headerlink" href="#incident-response-checklist" title="Permanent link">&para;</a></h3>
<ul>
<li>[ ] Incident identified (Slack alert, monitoring)</li>
<li>[ ] Severity assessed</li>
<li>[ ] Incident channel created</li>
<li>[ ] Initial investigation (logs, metrics, events)</li>
<li>[ ] Root cause identified</li>
<li>[ ] Mitigation applied</li>
<li>[ ] Verification of fix</li>
<li>[ ] Post-mortem scheduled</li>
<li>[ ] Documentation updated</li>
</ul>
<hr />
<p><strong>Last Updated</strong>: 2026-03-16
<strong>Maintained By</strong>: Platform Team
<strong>Emergency Contact</strong>: #platform-support on Slack</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "..", "features": ["navigation.instant", "navigation.sections", "navigation.top", "search.highlight", "content.code.copy"], "search": "../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../assets/javascripts/bundle.79ae519e.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink