Kubernetes / Nginx 金丝雀与灰度发布实战
Kubernetes / Nginx 金丝雀与灰度发布实战
面试指北 · 通用
📊 三种发布策略对比
| 发布策略 | 流量分配 | 风险等级 | 回滚速度 | 资源占用 | 适用场景 |
|---|---|---|---|---|---|
| 蓝绿发布 | 100% 切换 | 中等 | 秒级 | 2倍资源 | 关键业务系统 |
| 金丝雀发布 | 5%→20%→50%→100% | 低 | 分钟级 | 1.2倍资源 | 高可用要求 |
| 灰度发布 | 按用户群体 | 低 | 分钟级 | 1.5倍资源 | A/B测试场景 |
🐳 Docker + Nginx 实现方案
1. 蓝绿发布
Docker Compose 配置
# docker-compose-blue-green.yml
version: '3.8'
services:
# 蓝环境 (当前生产)
gis-app-blue:
image: gis-backend:v1.0
container_name: gis-blue
ports:
- "8081:8080"
environment:
- SPRING_PROFILES_ACTIVE=production
- DB_HOST=postgres-blue
networks:
- gis-network
labels:
- "version=blue"
- "env=production"
# 绿环境 (新版本)
gis-app-green:
image: gis-backend:v2.0
container_name: gis-green
ports:
- "8082:8080"
environment:
- SPRING_PROFILES_ACTIVE=production
- DB_HOST=postgres-green
networks:
- gis-network
labels:
- "version=green"
- "env=staging"
# Nginx 负载均衡器
nginx:
image: nginx:stable-alpine
container_name: nginx-lb
ports:
- "80:80"
- "443:443"
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf
- ./nginx/conf.d:/etc/nginx/conf.d
networks:
- gis-network
depends_on:
- gis-app-blue
- gis-app-green
networks:
gis-network:
driver: bridge
Nginx 蓝绿切换配置
# nginx/conf.d/blue-green.conf
upstream backend {
# 当前指向蓝环境
server gis-app-blue:8080 weight=100;
# 绿环境准备就绪但权重为0
server gis-app-green:8080 weight=0 backup;
}
server {
listen 80;
server_name gis.example.com;
# 健康检查端点
location /health {
access_log off;
return 200 "healthy\n";
add_header Content-Type text/plain;
}
# API流量代理
location /api/ {
proxy_pass http://backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# 超时配置
proxy_connect_timeout 5s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
}
# 静态资源
location / {
proxy_pass http://backend;
proxy_set_header Host $host;
}
}
# 管理端口 - 用于动态切换
server {
listen 8080;
server_name localhost;
# 切换到绿环境
location /switch-to-green {
access_log off;
content_by_lua_block {
-- 这里需要安装 lua 模块
-- 动态修改 upstream 配置
ngx.say("Switching to green environment...")
}
}
# 切换到蓝环境
location /switch-to-blue {
access_log off;
content_by_lua_block {
ngx.say("Switching to blue environment...")
}
}
}
蓝绿切换脚本
#!/bin/bash
# blue-green-deploy.sh
set -e
CURRENT_ENV=""
NEW_ENV=""
NGINX_CONTAINER="nginx-lb"
# 检查当前环境
check_current_env() {
CURRENT=$(docker exec $NGINX_CONTAINER nginx -T 2>/dev/null | grep "server gis-app-" | grep "weight=100" | head -1)
if echo "$CURRENT" | grep -q "blue"; then
CURRENT_ENV="blue"
NEW_ENV="green"
else
CURRENT_ENV="green"
NEW_ENV="blue"
fi
echo "当前环境: $CURRENT_ENV, 目标环境: $NEW_ENV"
}
# 健康检查
health_check() {
local env=$1
local port=""
if [ "$env" = "blue" ]; then
port="8081"
else
port="8082"
fi
echo "正在检查 $env 环境健康状态..."
for i in {1..30}; do
if curl -sf "http://localhost:$port/actuator/health" > /dev/null; then
echo "$env 环境健康检查通过"
return 0
fi
echo "等待 $env 环境启动... ($i/30)"
sleep 10
done
echo "$env 环境健康检查失败"
return 1
}
# 执行切换
switch_environment() {
echo "开始切换到 $NEW_ENV 环境..."
# 生成新的 nginx 配置
cat > ./nginx/conf.d/blue-green.conf << EOF
upstream backend {
server gis-app-$NEW_ENV:8080 weight=100;
server gis-app-$CURRENT_ENV:8080 weight=0 backup;
}
# ... 其余配置保持不变
EOF
# 重新加载 nginx 配置
docker exec $NGINX_CONTAINER nginx -s reload
echo "切换完成!当前环境: $NEW_ENV"
}
# 回滚操作
rollback() {
echo "正在回滚到 $CURRENT_ENV 环境..."
cat > ./nginx/conf.d/blue-green.conf << EOF
upstream backend {
server gis-app-$CURRENT_ENV:8080 weight=100;
server gis-app-$NEW_ENV:8080 weight=0 backup;
}
EOF
docker exec $NGINX_CONTAINER nginx -s reload
echo "回滚完成!"
}
# 主流程
main() {
check_current_env
# 启动新环境
docker-compose -f docker-compose-blue-green.yml up -d gis-app-$NEW_ENV
# 健康检查
if health_check $NEW_ENV; then
switch_environment
# 验证切换后的状态
sleep 5
if health_check $NEW_ENV; then
echo "部署成功!"
# 停止旧环境 (可选)
# docker-compose -f docker-compose-blue-green.yml stop gis-app-$CURRENT_ENV
else
echo "切换后健康检查失败,正在回滚..."
rollback
exit 1
fi
else
echo "新环境启动失败"
exit 1
fi
}
# 检查命令行参数
case "${1:-deploy}" in
deploy)
main
;;
rollback)
check_current_env
rollback
;;
status)
check_current_env
;;
*)
echo "Usage: $0 {deploy|rollback|status}"
exit 1
;;
esac
2. 金丝雀发布
Nginx 金丝雀配置
# nginx/conf.d/canary.conf
# 稳定版本
upstream stable {
server gis-app-stable:8080;
}
# 金丝雀版本
upstream canary {
server gis-app-canary:8080;
}
# 基于权重的金丝雀发布
upstream backend_weighted {
server gis-app-stable:8080 weight=95; # 95% 流量
server gis-app-canary:8080 weight=5; # 5% 流量
}
# 基于请求头的金丝雀发布
map $http_x_canary $pool {
default "stable";
"true" "canary";
}
# 基于 Cookie 的金丝雀发布
map $cookie_canary $canary_pool {
default "stable";
"enabled" "canary";
}
server {
listen 80;
server_name gis.example.com;
# 设置变量用于路由决策
set $upstream_pool "";
# 路由逻辑
access_by_lua_block {
local canary_header = ngx.var.http_x_canary
local canary_cookie = ngx.var.cookie_canary
local remote_addr = ngx.var.remote_addr
-- 优先级:请求头 > Cookie > IP 哈希
if canary_header == "true" then
ngx.var.upstream_pool = "canary"
elseif canary_cookie == "enabled" then
ngx.var.upstream_pool = "canary"
else
-- 基于 IP 的一致性哈希,5% 流量到金丝雀
local hash = ngx.crc32_long(remote_addr)
if hash % 100 < 5 then
ngx.var.upstream_pool = "canary"
else
ngx.var.upstream_pool = "stable"
end
end
}
location /api/ {
# 根据变量选择后端
proxy_pass http://$upstream_pool;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# 添加响应头标识版本
add_header X-Served-By $upstream_pool;
}
}
# 管理接口
server {
listen 8080;
server_name localhost;
# 调整金丝雀流量比例
location ~ ^/canary/weight/(\d+)$ {
content_by_lua_block {
local weight = tonumber(ngx.var[1])
if weight and weight >= 0 and weight <= 100 then
-- 动态调整权重配置
-- 这里需要实现动态更新 upstream 的逻辑
ngx.say("Canary weight set to: " .. weight .. "%")
else
ngx.status = 400
ngx.say("Invalid weight. Must be 0-100")
end
}
}
}
金丝雀发布自动化脚本
#!/bin/bash
# canary-deploy.sh
STABLE_IMAGE="gis-backend:v1.0"
CANARY_IMAGE="gis-backend:v2.0"
CANARY_WEIGHTS=(5 20 50 100) # 发布阶段的流量比例
STAGE_DURATION=300 # 每个阶段持续时间(秒)
deploy_canary() {
echo "开始金丝雀部署..."
# 启动金丝雀版本
docker run -d \
--name gis-app-canary \
--network gis-network \
-p 8083:8080 \
$CANARY_IMAGE
# 等待启动
sleep 30
# 健康检查
if ! curl -sf "http://localhost:8083/actuator/health"; then
echo "金丝雀版本启动失败"
cleanup_canary
exit 1
fi
echo "金丝雀版本启动成功"
}
update_weight() {
local weight=$1
echo "调整金丝雀流量到 $weight%"
# 更新 nginx 配置
sed -i "s/server gis-app-canary:8080 weight=[0-9]*/server gis-app-canary:8080 weight=$weight/" \
./nginx/conf.d/canary.conf
# 重新加载配置
docker exec nginx-lb nginx -s reload
}
monitor_metrics() {
local duration=$1
echo "监控指标 $duration 秒..."
local start_time=$(date +%s)
local end_time=$((start_time + duration))
while [ $(date +%s) -lt $end_time ]; do
# 检查错误率
local error_rate=$(curl -s "http://localhost:9090/api/v1/query?query=rate(http_requests_total{status=~\"5..\"}[5m])" | jq -r '.data.result[0].value[1] // 0')
# 检查响应时间
local avg_response_time=$(curl -s "http://localhost:9090/api/v1/query?query=rate(http_request_duration_seconds_sum[5m])/rate(http_request_duration_seconds_count[5m])" | jq -r '.data.result[0].value[1] // 0')
echo "错误率: $error_rate, 平均响应时间: ${avg_response_time}s"
# 如果指标异常,立即回滚
if (( $(echo "$error_rate > 0.01" | bc -l) )); then
echo "错误率过高,正在回滚..."
return 1
fi
sleep 30
done
return 0
}
cleanup_canary() {
echo "清理金丝雀环境..."
docker stop gis-app-canary 2>/dev/null || true
docker rm gis-app-canary 2>/dev/null || true
}
rollback() {
echo "正在回滚..."
update_weight 0
cleanup_canary
echo "回滚完成"
}
main() {
deploy_canary
for weight in "${CANARY_WEIGHTS[@]}"; do
update_weight $weight
if [ $weight -eq 100 ]; then
echo "金丝雀发布完成,切换到新版本"
# 停止旧版本
docker stop gis-app-stable
docker rename gis-app-canary gis-app-stable
break
fi
if ! monitor_metrics $STAGE_DURATION; then
rollback
exit 1
fi
echo "阶段 $weight% 成功,继续下一阶段"
done
}
case "${1:-deploy}" in
deploy)
main
;;
rollback)
rollback
;;
*)
echo "Usage: $0 {deploy|rollback}"
exit 1
;;
esac
☸️ Kubernetes 实现方案
1. 基于 Kubernetes Deployment 的蓝绿发布
应用部署清单
# k8s/blue-green/gis-app-blue.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: gis-app-blue
namespace: gis-system
labels:
app: gis-app
version: blue
spec:
replicas: 3
selector:
matchLabels:
app: gis-app
version: blue
template:
metadata:
labels:
app: gis-app
version: blue
spec:
containers:
- name: gis-app
image: gis-backend:v1.0
ports:
- containerPort: 8080
name: http
env:
- name: SPRING_PROFILES_ACTIVE
value: "production"
- name: VERSION
value: "blue"
resources:
requests:
memory: "1Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "1000m"
livenessProbe:
httpGet:
path: /actuator/health
port: 8080
initialDelaySeconds: 60
periodSeconds: 10
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8080
initialDelaySeconds: 30
periodSeconds: 5
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: gis-app-green
namespace: gis-system
labels:
app: gis-app
version: green
spec:
replicas: 0 # 初始为0,部署时扩容
selector:
matchLabels:
app: gis-app
version: green
template:
metadata:
labels:
app: gis-app
version: green
spec:
containers:
- name: gis-app
image: gis-backend:v2.0
ports:
- containerPort: 8080
name: http
env:
- name: SPRING_PROFILES_ACTIVE
value: "production"
- name: VERSION
value: "green"
resources:
requests:
memory: "1Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "1000m"
livenessProbe:
httpGet:
path: /actuator/health
port: 8080
initialDelaySeconds: 60
periodSeconds: 10
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8080
initialDelaySeconds: 30
periodSeconds: 5
Service 配置
# k8s/blue-green/service.yaml
apiVersion: v1
kind: Service
metadata:
name: gis-app-service
namespace: gis-system
spec:
selector:
app: gis-app
version: blue # 初始指向蓝环境
ports:
- port: 80
targetPort: 8080
protocol: TCP
name: http
type: ClusterIP
---
# 用于蓝绿切换的服务
apiVersion: v1
kind: Service
metadata:
name: gis-app-blue
namespace: gis-system
spec:
selector:
app: gis-app
version: blue
ports:
- port: 80
targetPort: 8080
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: gis-app-green
namespace: gis-system
spec:
selector:
app: gis-app
version: green
ports:
- port: 80
targetPort: 8080
protocol: TCP
Nginx Ingress 配置
# k8s/blue-green/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gis-app-ingress
namespace: gis-system
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/rewrite-target: /
nginx.ingress.kubernetes.io/ssl-redirect: "false"
# 自定义 Nginx 配置
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "X-Served-By: $service_name";
more_set_headers "X-Pod-Name: $hostname";
spec:
rules:
- host: gis.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: gis-app-service
port:
number: 80
---
# 管理入口,用于蓝绿切换
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: gis-app-admin
namespace: gis-system
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/rewrite-target: /
spec:
rules:
- host: gis-admin.example.com
http:
paths:
- path: /blue
pathType: Prefix
backend:
service:
name: gis-app-blue
port:
number: 80
- path: /green
pathType: Prefix
backend:
service:
name: gis-app-green
port:
number: 80
蓝绿发布自动化脚本(K8s版)
#!/bin/bash
# k8s-blue-green-deploy.sh
NAMESPACE="gis-system"
APP_NAME="gis-app"
NEW_IMAGE="$1"
if [ -z "$NEW_IMAGE" ]; then
echo "Usage: $0 <new-image>"
exit 1
fi
# 获取当前活动环境
get_current_env() {
local current_selector=$(kubectl get service gis-app-service -n $NAMESPACE -o jsonpath='{.spec.selector.version}')
echo $current_selector
}
# 获取目标环境
get_target_env() {
local current=$1
if [ "$current" = "blue" ]; then
echo "green"
else
echo "blue"
fi
}
# 部署新版本
deploy_new_version() {
local target_env=$1
local image=$2
echo "正在部署 $target_env 环境,镜像: $image"
# 更新镜像
kubectl set image deployment/gis-app-$target_env gis-app=$image -n $NAMESPACE
# 扩容到3个副本
kubectl scale deployment gis-app-$target_env --replicas=3 -n $NAMESPACE
# 等待就绪
kubectl wait --for=condition=available --timeout=600s deployment/gis-app-$target_env -n $NAMESPACE
if [ $? -ne 0 ]; then
echo "$target_env 环境部署失败"
return 1
fi
echo "$target_env 环境部署成功"
return 0
}
# 健康检查
health_check() {
local env=$1
echo "正在进行 $env 环境健康检查..."
# 通过服务进行健康检查
for i in {1..30}; do
if kubectl exec -n $NAMESPACE deployment/gis-app-$env -- curl -sf http://localhost:8080/actuator/health > /dev/null 2>&1; then
echo "$env 环境健康检查通过"
return 0
fi
echo "等待 $env 环境就绪... ($i/30)"
sleep 10
done
echo "$env 环境健康检查失败"
return 1
}
# 切换流量
switch_traffic() {
local target_env=$1
echo "正在切换流量到 $target_env 环境..."
kubectl patch service gis-app-service -n $NAMESPACE -p '{"spec":{"selector":{"version":"'$target_env'"}}}'
if [ $? -eq 0 ]; then
echo "流量切换成功"
return 0
else
echo "流量切换失败"
return 1
fi
}
# 清理旧环境
cleanup_old_env() {
local old_env=$1
echo "正在缩容 $old_env 环境..."
kubectl scale deployment gis-app-$old_env --replicas=0 -n $NAMESPACE
echo "$old_env 环境已缩容"
}
# 回滚操作
rollback() {
local original_env=$1
echo "正在回滚到 $original_env 环境..."
switch_traffic $original_env
echo "回滚完成"
}
# 主流程
main() {
local current_env=$(get_current_env)
local target_env=$(get_target_env $current_env)
echo "当前环境: $current_env"
echo "目标环境: $target_env"
echo "新镜像: $NEW_IMAGE"
# 部署新版本
if ! deploy_new_version $target_env $NEW_IMAGE; then
echo "部署失败"
exit 1
fi
# 健康检查
if ! health_check $target_env; then
echo "健康检查失败,正在清理..."
kubectl scale deployment gis-app-$target_env --replicas=0 -n $NAMESPACE
exit 1
fi
# 切换流量
if ! switch_traffic $target_env; then
echo "流量切换失败,正在回滚..."
rollback $current_env
exit 1
fi
# 验证切换后状态
sleep 10
if health_check $target_env; then
echo "蓝绿部署成功!"
# 等待5分钟后清理旧环境
echo "5分钟后将清理 $current_env 环境..."
sleep 300
cleanup_old_env $current_env
else
echo "切换后验证失败,正在回滚..."
rollback $current_env
exit 1
fi
}
# 执行主流程
main
2. 基于 Istio 的金丝雀发布
VirtualService 配置
# k8s/canary/virtual-service.yaml
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: gis-app
namespace: gis-system
spec:
hosts:
- gis.example.com
http:
# 基于请求头的路由
- match:
- headers:
x-canary:
exact: "true"
route:
- destination:
host: gis-app-service
subset: canary
# 基于权重的路由
- route:
- destination:
host: gis-app-service
subset: stable
weight: 95
- destination:
host: gis-app-service
subset: canary
weight: 5
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: gis-app
namespace: gis-system
spec:
host: gis-app-service
subsets:
- name: stable
labels:
version: stable
- name: canary
labels:
version: canary
trafficPolicy:
connectionPool:
tcp:
maxConnections: 100
http:
http1MaxPendingRequests: 10
maxRequestsPerConnection: 2
outlierDetection:
consecutiveErrors: 3
interval: 30s
baseEjectionTime: 30s
Flagger 自动化金丝雀发布
# k8s/canary/flagger-canary.yaml
apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
name: gis-app
namespace: gis-system
spec:
# 目标部署
targetRef:
apiVersion: apps/v1
kind: Deployment
name: gis-app
# 服务配置
service:
port: 80
targetPort: 8080
gateways:
- gis-gateway
hosts:
- gis.example.com
# 分析配置
analysis:
# 分析间隔
interval: 1m
# 分析阈值
threshold: 5
# 最大权重
maxWeight: 50
# 权重增长步长
stepWeight: 10
# 成功率阈值
metrics:
- name: request-success-rate
thresholdRange:
min: 99
interval: 1m
- name: request-duration
thresholdRange:
max: 500
interval: 1m
# Webhook 测试
webhooks:
- name: load-test
url: http://flagger-loadtester.test-system/
timeout: 5s
metadata:
cmd: "hey -z 1m -q 10 -c 2 http://gis.example.com/api/health"
📊 监控和告警配置
Prometheus 监控规则
# monitoring/prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: gis-app-canary
namespace: gis-system
spec:
groups:
- name: gis-app.rules
rules:
# 错误率告警
- alert: HighErrorRate
expr: |
(
rate(http_requests_total{job="gis-app",status=~"5.."}[5m])
/
rate(http_requests_total{job="gis-app"}[5m])
) > 0.01
for: 2m
labels:
severity: critical
annotations:
summary: "GIS应用错误率过高"
description: "{{ $labels.version }} 版本错误率 {{ $value | humanizePercentage }}"
# 响应时间告警
- alert: HighLatency
expr: |
histogram_quantile(0.95,
rate(http_request_duration_seconds_bucket{job="gis-app"}[5m])
) > 1
for: 5m
labels:
severity: warning
annotations:
summary: "GIS应用响应时间过长"
description: "{{ $labels.version }} 版本 95% 响应时间超过 1 秒"
Grafana 监控大盘
{
"dashboard": {
"title": "GIS Application Canary Deployment",
"panels": [
{
"title": "Request Rate by Version",
"type": "graph",
"targets": [
{
"expr": "rate(http_requests_total{job=\"gis-app\"}[5m])",
"legendFormat": "{{version}} - {{status}}"
}
]
},
{
"title": "Error Rate by Version",
"type": "graph",
"targets": [
{
"expr": "rate(http_requests_total{job=\"gis-app\",status=~\"5..\"}[5m]) / rate(http_requests_total{job=\"gis-app\"}[5m])",
"legendFormat": "{{version}} Error Rate"
}
]
},
{
"title": "Response Time P95",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket{job=\"gis-app\"}[5m]))",
"legendFormat": "{{version}} P95"
}
]
}
]
}
}
🔄 CI/CD 集成
GitLab CI 配置
# .gitlab-ci.yml
stages:
- build
- test
- deploy-canary
- promote
variables:
DOCKER_REGISTRY: "registry.example.com"
APP_NAME: "gis-backend"
KUBE_NAMESPACE: "gis-system"
build:
stage: build
script:
- docker build -t $DOCKER_REGISTRY/$APP_NAME:$CI_COMMIT_SHA .
- docker push $DOCKER_REGISTRY/$APP_NAME:$CI_COMMIT_SHA
deploy-canary:
stage: deploy-canary
script:
- ./scripts/k8s-canary-deploy.sh $DOCKER_REGISTRY/$APP_NAME:$CI_COMMIT_SHA
environment:
name: production-canary
when: manual
promote-production:
stage: promote
script:
- ./scripts/k8s-promote-canary.sh
environment:
name: production
when: manual
dependencies:
- deploy-canary
🎯 最佳实践总结
选择建议
- 小团队/简单应用: Docker + Nginx
- 中大型团队: Kubernetes + Ingress Controller
- 微服务架构: Istio + Flagger
- 高可用要求: 蓝绿发布
- 渐进式验证: 金丝雀发布
关键要点
- 健康检查: 完善的应用健康检查接口
- 监控告警: 实时监控关键指标
- 自动回滚: 检测到异常时自动回滚
- 流量控制: 精确的流量分配和切换
- 测试策略: 自动化测试验证新版本
这套方案可以根据你们的具体需求进行调整和优化!
License:
CC BY 4.0