Release Management
Best practices for shipping software reliably through structured release processes, deployment strategies, and rollback automation.
Deployment Strategy Comparison
Choosing the right deployment strategy depends on risk tolerance, infrastructure budget, and rollback requirements.
| Strategy | Zero Downtime | Rollback Speed | Infrastructure Cost | Complexity | Best For | |----------|--------------|----------------|--------------------:|------------|----------| | Blue-Green | Yes | Instant (switch) | 2x | Medium | Critical services, compliance | | Canary | Yes | Fast (route away) | 1.05-1.5x | High | High-traffic user-facing apps | | Rolling | Yes | Slow (re-deploy) | 1x | Low | Stateless microservices | | Recreate | No (brief outage) | Slow (re-deploy) | 1x | Low | Dev/staging, batch jobs | | A/B Testing | Yes | Fast (route away) | 1.2-1.5x | High | Feature experimentation | | Shadow/Dark | Yes | N/A (no user impact) | 1.5-2x | Very High | ML models, data pipelines |
Decision Matrix
Is the service stateful?
YES --> Can you afford 2x infrastructure?
YES --> Blue-Green
NO --> Rolling (with drain + health checks)
NO --> Is it high-traffic (>1000 rps)?
YES --> Canary (gradual rollout)
NO --> Rolling (simple, cost-effective)Canary Deployment Configuration
Kubernetes Canary with Argo Rollouts
# argo-rollout.yaml -- Progressive canary with automated analysis
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
name: payment-service
namespace: production
spec:
replicas: 10
revisionHistoryLimit: 3
selector:
matchLabels:
app: payment-service
strategy:
canary:
# Canary traffic steps with pause for analysis
steps:
- setWeight: 5
- pause: { duration: 5m } # 5% for 5 minutes
- analysis:
templates:
- templateName: canary-success-rate
args:
- name: service-name
value: payment-service
- setWeight: 20
- pause: { duration: 10m } # 20% for 10 minutes
- analysis:
templates:
- templateName: canary-success-rate
- setWeight: 50
- pause: { duration: 10m } # 50% for 10 minutes
- setWeight: 80
- pause: { duration: 5m } # 80% for 5 minutes
# 100% happens automatically after final step
# Auto-rollback on failure
abortScaleDownDelaySeconds: 30
dynamicStableScale: true
# Traffic management via Istio
trafficRouting:
istio:
virtualServices:
- name: payment-service-vsvc
routes:
- primary
destinationRule:
name: payment-service-destrule
canarySubsetName: canary
stableSubsetName: stable
# Analysis template for canary health
analysis:
templates:
- templateName: canary-success-rate
startingStep: 2
args:
- name: service-name
value: payment-service
template:
metadata:
labels:
app: payment-service
spec:
containers:
- name: payment-service
image: registry.example.com/payment-service:v2.3.1
ports:
- containerPort: 8080
readinessProbe:
httpGet:
path: /healthz
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
resources:
requests:
cpu: 250m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
---
# Analysis template -- Prometheus-based success rate check
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
name: canary-success-rate
spec:
args:
- name: service-name
metrics:
- name: success-rate
interval: 60s
count: 5
successCondition: result[0] >= 0.99
failureLimit: 2
provider:
prometheus:
address: http://prometheus.monitoring:9090
query: |
sum(rate(http_requests_total{
service="{{args.service-name}}",
status=~"2..",
canary="true"
}[2m])) /
sum(rate(http_requests_total{
service="{{args.service-name}}",
canary="true"
}[2m]))
- name: p99-latency
interval: 60s
count: 5
successCondition: result[0] <= 500
failureLimit: 2
provider:
prometheus:
address: http://prometheus.monitoring:9090
query: |
histogram_quantile(0.99,
sum(rate(http_request_duration_ms_bucket{
service="{{args.service-name}}",
canary="true"
}[2m])) by (le)
)Feature Flag Implementation
LaunchDarkly Pattern with Gradual Rollout
// feature-flags.ts -- Structured feature flag management
import * as LaunchDarkly from "@launchdarkly/node-server-sdk";
// --- Flag Configuration Types ---
interface FlagConfig {
key: string;
description: string;
type: "boolean" | "multivariate" | "percentage";
owner: string; // Team responsible
createdAt: string; // For cleanup tracking
maxAge: string; // Expected lifetime: "temporary" | "permanent"
cleanupTicket?: string; // JIRA ticket to remove flag
}
// Registry prevents flag sprawl -- all flags must be declared
const FLAG_REGISTRY: Record<string, FlagConfig> = {
"new-checkout-flow": {
key: "new-checkout-flow",
description: "Redesigned checkout with one-page form",
type: "percentage",
owner: "checkout-team",
createdAt: "2025-11-01",
maxAge: "temporary",
cleanupTicket: "CHECKOUT-1234",
},
"payment-v2-api": {
key: "payment-v2-api",
description: "Route payments through v2 processor",
type: "boolean",
owner: "payments-team",
createdAt: "2025-10-15",
maxAge: "temporary",
cleanupTicket: "PAY-567",
},
};
// --- Client Initialization ---
let ldClient: LaunchDarkly.LDClient;
export async function initFeatureFlags(): Promise<void> {
ldClient = LaunchDarkly.init(process.env.LAUNCHDARKLY_SDK_KEY!);
await ldClient.waitForInitialization({ timeout: 10 });
console.log("LaunchDarkly client initialized");
}
// --- Flag Evaluation with Fallback ---
export async function isEnabled(
flagKey: string,
context: LaunchDarkly.LDContext,
defaultValue = false
): Promise<boolean> {
// Validate flag is registered
if (!FLAG_REGISTRY[flagKey]) {
console.warn(`Unknown flag: ${flagKey}. Returning default.`);
return defaultValue;
}
try {
const value = await ldClient.variation(flagKey, context, defaultValue);
return Boolean(value);
} catch (err) {
// Flag evaluation failure should never break the app
console.error(`Flag evaluation failed for ${flagKey}:`, err);
return defaultValue;
}
}
// --- Usage in Route Handler ---
export async function checkoutHandler(req: Request, res: Response) {
const userContext: LaunchDarkly.LDContext = {
kind: "user",
key: req.user.id,
email: req.user.email,
custom: {
plan: req.user.plan, // Target by plan tier
region: req.user.region, // Target by geography
company: req.user.orgId, // Target by organization
},
};
const useNewCheckout = await isEnabled("new-checkout-flow", userContext);
if (useNewCheckout) {
return renderNewCheckout(req, res);
}
return renderLegacyCheckout(req, res);
}
// --- Stale Flag Detection ---
export function getStaleFlags(maxAgeDays = 90): FlagConfig[] {
const now = Date.now();
return Object.values(FLAG_REGISTRY).filter((flag) => {
if (flag.maxAge === "permanent") return false;
const age = now - new Date(flag.createdAt).getTime();
return age > maxAgeDays * 24 * 60 * 60 * 1000;
});
}