【网络工程】Envoy 架构剖析：xDS、Filter Chain 与热重启

# Envoy Worker 配置
# envoy.yaml
admin:
  address:
    socket_address:
      address: 0.0.0.0
      port_value: 9901

# Worker 数自动设置为 CPU 核心数
# 也可以通过 --concurrency 参数指定
# envoy --concurrency 4

# 查看 Envoy 线程
ps -T -p $(pgrep envoy) | head -10
# PID    SPID   TTY   TIME CMD
# 1234   1234   ?     00:00:01 envoy         (main thread)
# 1234   1235   ?     00:00:05 envoy         (worker 0)
# 1234   1236   ?     00:00:05 envoy         (worker 1)
# 1234   1237   ?     00:00:05 envoy         (worker 2)
# 1234   1238   ?     00:00:05 envoy         (worker 3)

# 查看各线程 CPU 使用
top -H -p $(pgrep envoy)

# envoy.yaml — 完整的 L7 代理配置
static_resources:
  listeners:
    - name: http_listener
      address:
        socket_address:
          address: 0.0.0.0
          port_value: 8080
      filter_chains:
        - filters:
            # L4 Filter: HTTP Connection Manager
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: ingress_http
                codec_type: AUTO

                # L7 Filter Chain
                http_filters:
                  # 1. CORS
                  - name: envoy.filters.http.cors
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors

                  # 2. 限流
                  - name: envoy.filters.http.local_ratelimit
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit
                      stat_prefix: http_local_rate_limiter
                      token_bucket:
                        max_tokens: 1000
                        tokens_per_fill: 100
                        fill_interval: 1s

                  # 3. JWT 认证
                  - name: envoy.filters.http.jwt_authn
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication
                      providers:
                        auth0:
                          issuer: https://auth.example.com/
                          audiences: ["api.example.com"]
                          remote_jwks:
                            http_uri:
                              uri: https://auth.example.com/.well-known/jwks.json
                              cluster: auth_cluster
                              timeout: 5s
                            cache_duration: 600s
                      rules:
                        - match:
                            prefix: /api/
                          requires:
                            provider_name: auth0

                  # 4. 路由（必须是最后一个）
                  - name: envoy.filters.http.router
                    typed_config:
                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router

                # 路由配置
                route_config:
                  name: local_route
                  virtual_hosts:
                    - name: app
                      domains: ["*"]
                      routes:
                        - match:
                            prefix: "/api/v2/"
                          route:
                            cluster: api_v2
                            timeout: 30s
                            retry_policy:
                              retry_on: "5xx,reset,connect-failure"
                              num_retries: 2
                              per_try_timeout: 10s
                        - match:
                            prefix: "/api/"
                          route:
                            cluster: api_v1
                            timeout: 30s
                        - match:
                            prefix: "/"
                          route:
                            cluster: web_cluster
                            timeout: 15s

  clusters:
    - name: api_v2
      type: STRICT_DNS
      connect_timeout: 5s
      lb_policy: LEAST_REQUEST
      load_assignment:
        cluster_name: api_v2
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: api-v2.internal
                      port_value: 8080
      # 上游连接池
      circuit_breakers:
        thresholds:
          - max_connections: 1024
            max_pending_requests: 1024
            max_requests: 1024
            max_retries: 3

    - name: api_v1
      type: STRICT_DNS
      connect_timeout: 5s
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: api_v1
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: api-v1.internal
                      port_value: 8080

    - name: web_cluster
      type: STRICT_DNS
      connect_timeout: 5s
      lb_policy: ROUND_ROBIN
      load_assignment:
        cluster_name: web_cluster
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: web.internal
                      port_value: 8080

# Wasm Filter 配置
http_filters:
  - name: envoy.filters.http.wasm
    typed_config:
      "@type": type.googleapis.com/udpa.type.v1.TypedStruct
      type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
      value:
        config:
          name: "custom_auth"
          root_id: "custom_auth_root"
          vm_config:
            runtime: "envoy.wasm.runtime.v8"
            code:
              local:
                filename: "/etc/envoy/filters/custom_auth.wasm"
          configuration:
            "@type": type.googleapis.com/google.protobuf.StringValue
            value: |
              {"auth_header": "X-API-Key", "valid_keys": ["key1", "key2"]}

// Rust Wasm Filter 示例（使用 proxy-wasm SDK）
use proxy_wasm::traits::*;
use proxy_wasm::types::*;

proxy_wasm::main! {{
    proxy_wasm::set_http_context(|_, _| -> Box<dyn HttpContext> {
        Box::new(AuthFilter)
    });
}}

struct AuthFilter;

impl Context for AuthFilter {}

impl HttpContext for AuthFilter {
    fn on_http_request_headers(&mut self, _: usize, _: bool) -> Action {
        // 检查 API Key
        match self.get_http_request_header("X-API-Key") {
            Some(key) if key == "valid-key-123" => {
                Action::Continue
            }
            _ => {
                self.send_http_response(
                    401,
                    vec![("Content-Type", "application/json")],
                    Some(b"{\"error\": \"unauthorized\"}"),
                );
                Action::Pause
            }
        }
    }
}

# envoy.yaml — 使用 xDS 动态配置
node:
  id: "envoy-node-1"
  cluster: "web-cluster"

dynamic_resources:
  # LDS: 动态发现 Listener
  lds_config:
    resource_api_version: V3
    api_config_source:
      api_type: GRPC
      transport_api_version: V3
      grpc_services:
        - envoy_grpc:
            cluster_name: xds_cluster

  # CDS: 动态发现 Cluster
  cds_config:
    resource_api_version: V3
    api_config_source:
      api_type: GRPC
      transport_api_version: V3
      grpc_services:
        - envoy_grpc:
            cluster_name: xds_cluster

static_resources:
  clusters:
    # xDS 控制面集群（这个必须是静态配置）
    - name: xds_cluster
      type: STRICT_DNS
      connect_timeout: 5s
      typed_extension_protocol_options:
        envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
          "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
          explicit_http_config:
            http2_protocol_options: {}
      load_assignment:
        cluster_name: xds_cluster
        endpoints:
          - lb_endpoints:
              - endpoint:
                  address:
                    socket_address:
                      address: control-plane.internal
                      port_value: 18000

# 查看 Envoy 当前的动态配置
curl -s http://localhost:9901/config_dump | jq '.configs[] | .["@type"]'
# "type.googleapis.com/envoy.admin.v3.BootstrapConfigDump"
# "type.googleapis.com/envoy.admin.v3.ClustersConfigDump"
# "type.googleapis.com/envoy.admin.v3.ListenersConfigDump"
# "type.googleapis.com/envoy.admin.v3.RoutesConfigDump"

# 查看 xDS 版本信息
curl -s http://localhost:9901/config_dump?resource=dynamic_listeners | \
    jq '.configs[0].dynamic_listeners[].active_state.version_info'

// 简单的 xDS 控制面实现
package main

import (
    "context"
    "log"
    "net"

    clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
    corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
    endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
    "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
    "github.com/envoyproxy/go-control-plane/pkg/server/v3"
    "google.golang.org/grpc"

    discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
)

func main() {
    // 创建配置缓存
    snapshotCache := cache.NewSnapshotCache(false, cache.IDHash{}, nil)

    // 创建配置快照
    snapshot, _ := cache.NewSnapshot("v1",
        map[string][]cache.Resource{
            // Cluster 配置
            "type.googleapis.com/envoy.config.cluster.v3.Cluster": {
                &clusterv3.Cluster{
                    Name: "app_cluster",
                    ClusterDiscoveryType: &clusterv3.Cluster_Type{
                        Type: clusterv3.Cluster_STRICT_DNS,
                    },
                    LoadAssignment: &endpointv3.ClusterLoadAssignment{
                        ClusterName: "app_cluster",
                        Endpoints: []*endpointv3.LocalityLbEndpoints{{
                            LbEndpoints: []*endpointv3.LbEndpoint{{
                                HostIdentifier: &endpointv3.LbEndpoint_Endpoint{
                                    Endpoint: &endpointv3.Endpoint{
                                        Address: &corev3.Address{
                                            Address: &corev3.Address_SocketAddress{
                                                SocketAddress: &corev3.SocketAddress{
                                                    Address: "10.0.1.1",
                                                    PortSpecifier: &corev3.SocketAddress_PortValue{
                                                        PortValue: 8080,
                                                    },
                                                },
                                            },
                                        },
                                    },
                                },
                            }},
                        }},
                    },
                },
            },
        },
    )

    // 设置快照
    snapshotCache.SetSnapshot(context.Background(), "envoy-node-1", snapshot)

    // 启动 gRPC 服务器
    srv := server.NewServer(context.Background(), snapshotCache, nil)
    grpcServer := grpc.NewServer()
    discoveryv3.RegisterAggregatedDiscoveryServiceServer(grpcServer, srv)

    lis, _ := net.Listen("tcp", ":18000")
    log.Println("xDS 控制面启动在 :18000")
    grpcServer.Serve(lis)
}

# 热重启参数
envoy -c /etc/envoy/envoy.yaml \
    --restart-epoch 0 \
    --hot-restart-version $(envoy --hot-restart-version)

# 触发热重启
# 启动新进程（epoch + 1）
envoy -c /etc/envoy/envoy-new.yaml \
    --restart-epoch 1 \
    --drain-time-s 60 \
    --parent-shutdown-time-s 90

# drain-time-s: 旧进程的排水期（停止接受新连接后等待的时间）
# parent-shutdown-time-s: 旧进程的最大存活时间

# 查看热重启状态
curl -s http://localhost:9901/hot_restart_version
# 返回热重启兼容版本号

# 查看当前 epoch
curl -s http://localhost:9901/server_info | jq '.hot_restart_version'

# Istio Sidecar 中的 Envoy 热重启
# pilot-agent 管理 Envoy 的生命周期
# 当配置变化时:
# 1. pilot-agent 接收新配置
# 2. pilot-agent 通过 xDS 推送给 Envoy
# 3. Envoy 的 xDS 更新机制处理（不需要热重启）
# 4. 只有二进制升级时才需要热重启

# 手动触发 Sidecar Envoy 的热重启
kubectl exec <pod> -c istio-proxy -- \
    pilot-agent request POST /quitquitquit
# Kubelet 会重新启动 Sidecar 容器

# 查看所有统计指标
curl -s http://localhost:9901/stats

# 按前缀过滤
curl -s http://localhost:9901/stats?filter=cluster.api_v2

# Prometheus 格式导出
curl -s http://localhost:9901/stats/prometheus

# 关键指标:
# cluster.{name}.upstream_rq_total          — 总请求数
# cluster.{name}.upstream_rq_2xx            — 2xx 响应数
# cluster.{name}.upstream_rq_5xx            — 5xx 响应数
# cluster.{name}.upstream_cx_active         — 活跃连接数
# cluster.{name}.upstream_cx_connect_fail   — 连接失败数
# cluster.{name}.upstream_rq_timeout        — 请求超时数
# cluster.{name}.upstream_rq_retry          — 重试次数
# cluster.{name}.upstream_rq_pending_active — 等待连接的请求数
# http.{stat_prefix}.downstream_rq_total    — 下游总请求数
# listener.{address}.downstream_cx_active   — 监听器活跃连接数

# 配置链路追踪（Zipkin）
tracing:
  http:
    name: envoy.tracers.zipkin
    typed_config:
      "@type": type.googleapis.com/envoy.config.trace.v3.ZipkinConfig
      collector_cluster: zipkin
      collector_endpoint: "/api/v2/spans"
      collector_endpoint_version: HTTP_JSON

# HTTP Connection Manager 中启用追踪
http_connection_manager:
  tracing:
    provider:
      name: envoy.tracers.zipkin
    random_sampling:
      value: 1.0    # 100% 采样（生产环境调低）

# 访问日志配置
http_connection_manager:
  access_log:
    - name: envoy.access_loggers.file
      typed_config:
        "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
        path: /var/log/envoy/access.log
        log_format:
          json_format:
            timestamp: "%START_TIME%"
            method: "%REQ(:METHOD)%"
            path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
            protocol: "%PROTOCOL%"
            response_code: "%RESPONSE_CODE%"
            response_flags: "%RESPONSE_FLAGS%"
            upstream_host: "%UPSTREAM_HOST%"
            upstream_service_time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"
            duration: "%DURATION%"
            request_id: "%REQ(X-REQUEST-ID)%"
            upstream_cluster: "%UPSTREAM_CLUSTER%"

# Envoy Response Flags 含义
# UH: 上游不健康（无可用后端）
# UF: 上游连接失败
# UO: 上游溢出（熔断器触发）
# UT: 上游请求超时
# LR: 本地重置（Envoy 重置连接）
# UR: 上游重置
# UC: 上游连接终止
# NR: 无路由匹配
# RL: 速率限制
# DC: 下游连接终止
# DPE: 下游协议错误

# 分析 Response Flags
cat /var/log/envoy/access.log | \
    jq -r '.response_flags' | sort | uniq -c | sort -rn

clusters:
  - name: api_cluster
    circuit_breakers:
      thresholds:
        - priority: DEFAULT
          max_connections: 1024       # 最大连接数
          max_pending_requests: 1024  # 等待连接的最大请求数
          max_requests: 1024          # 最大并发请求数
          max_retries: 3              # 最大并发重试数
          track_remaining: true       # 暴露剩余配额指标

# 监控熔断器状态
curl -s http://localhost:9901/stats | grep circuit_breakers
# cluster.api_cluster.circuit_breakers.default.cx_open: 0
# cluster.api_cluster.circuit_breakers.default.rq_open: 0
# cluster.api_cluster.circuit_breakers.default.cx_pool_open: 0
# cluster.api_cluster.circuit_breakers.default.remaining_cx: 1024
# cluster.api_cluster.circuit_breakers.default.remaining_rq: 1024

clusters:
  - name: api_cluster
    outlier_detection:
      consecutive_5xx: 5              # 连续 5 个 5xx 后驱逐
      interval: 10s                   # 检测间隔
      base_ejection_time: 30s         # 基础驱逐时间
      max_ejection_percent: 50        # 最大驱逐比例
      enforcing_consecutive_5xx: 100  # 执行比例（100%）
      success_rate_minimum_hosts: 3   # 成功率统计最小主机数
      success_rate_request_volume: 100 # 成功率统计最小请求数
      success_rate_stdev_factor: 1900 # 成功率偏差因子

# Envoy Admin API 常用端点

# 服务器信息
curl http://localhost:9901/server_info | jq .

# 集群状态
curl http://localhost:9901/clusters
# 显示每个集群的所有端点及其健康状态

# 配置转储
curl http://localhost:9901/config_dump | jq .

# 日志级别动态调整
curl -X POST http://localhost:9901/logging?level=debug
curl -X POST http://localhost:9901/logging?upstream=trace

# 重置统计
curl -X POST http://localhost:9901/reset_counters

# 排水（准备关闭）
curl -X POST http://localhost:9901/drain_listeners

# 健康检查失败（从 LB 中摘除自己）
curl -X POST http://localhost:9901/healthcheck/fail

# 准备就绪检查
curl http://localhost:9901/ready
# 返回 200 = Envoy 就绪，返回 503 = 未就绪