【Kubernetes 网络深度系列】Cilium 身份识别与零信任网络

$ kubectl get pod -o wide
NAME                    READY   IP            NODE
frontend-abc123-xk9z2   1/1    10.244.1.47   node-1

# 滚动更新后，IP 完全变化
$ kubectl rollout restart deploy/frontend
$ kubectl get pod -o wide
NAME                    READY   IP            NODE
frontend-abc123-j7q4n   1/1    10.244.1.52   node-1

$ kubectl get cep -n default frontend-abc123-xk9z2 -o yaml
apiVersion: cilium.io/v2
kind: CiliumEndpoint
metadata:
  name: frontend-abc123-xk9z2
status:
  identity:
    id: 48372
    labels:
      - k8s:app=frontend
      - k8s:env=prod
      - k8s:io.kubernetes.pod.namespace=default

# Cilium ConfigMap
labels: "k8s:io.kubernetes.pod.namespace k8s:app k8s:env"

$ kubectl get ciliumidentity 48372 -o yaml
apiVersion: cilium.io/v2
kind: CiliumIdentity
metadata:
  name: "48372"
security-labels:
  k8s:app: frontend
  k8s:env: prod
  k8s:io.kubernetes.pod.namespace: default

// bpf/lib/identity.h（简化）
static __always_inline __u32 lookup_ip4_remote_identity(__be32 ip)
{
    struct ipcache_key key = {
        .lpm_key.prefixlen = 32,
        .family = ENDPOINT_KEY_IPV4,
        .ip4 = ip,
    };
    struct remote_endpoint_info *info;
    info = map_lookup_elem(&IPCACHE_MAP, &key);
    if (info)
        return info->sec_identity;
    return WORLD_ID;
}

// bpf/lib/conntrack.h（简化）
struct ct_entry {
    __u64 rx_packets;
    __u64 tx_packets;
    __u32 lifetime;
    __u16 flags;
    __u32 src_sec_id;     // 源 Identity 缓存
    __u32 ifindex;
};

$ cilium clustermesh enable --service-type LoadBalancer
$ cilium clustermesh connect --destination-context cluster-2

cluster:
  name: cluster-1
  id: 1        # 范围：1-255

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: allow-cross-cluster-frontend
spec:
  endpointSelector:
    matchLabels:
      app: backend
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
            io.cilium.k8s.policy.cluster: cluster-1
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: l7-http-policy
spec:
  endpointSelector:
    matchLabels:
      app: api-server
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
          rules:
            http:
              - method: GET
                path: "/api/v1/.*"
              - method: POST
                path: "/api/v1/orders"
                headers:
                  - 'Content-Type: application/json'

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: l7-grpc-policy
spec:
  endpointSelector:
    matchLabels:
      app: user-service
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: api-gateway
      toPorts:
        - ports:
            - port: "9090"
              protocol: TCP
          rules:
            http:
              - method: POST
                path: "/user.UserService/GetUser"
              - method: POST
                path: "/user.UserService/ListUsers"

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: l7-kafka-policy
spec:
  endpointSelector:
    matchLabels:
      app: kafka-broker
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: event-producer
      toPorts:
        - ports:
            - port: "9092"
              protocol: TCP
          rules:
            kafka:
              - apiKey: "produce"
                topic: "user-events"
              - apiKey: "metadata"
    - fromEndpoints:
        - matchLabels:
            app: event-consumer
      toPorts:
        - ports:
            - port: "9092"
              protocol: TCP
          rules:
            kafka:
              - apiKey: "fetch"
                topic: "user-events"
              - apiKey: "offsets"
              - apiKey: "metadata"

$ helm upgrade cilium cilium/cilium \
    --set authentication.mutual.spire.enabled=true \
    --set authentication.mutual.spire.install.enabled=true

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: require-mtls
spec:
  endpointSelector:
    matchLabels:
      app: payment-service
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: order-service
      authentication:
        mode: required
      toPorts:
        - ports:
            - port: "8443"
              protocol: TCP

apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
  name: default-deny-all
spec:
  endpointSelector: {}
  ingress:
    - fromEntities:
        - cluster
  egress:
    - toEntities:
        - cluster
    - toEndpoints:
        - matchLabels:
            io.kubernetes.pod.namespace: kube-system
            k8s-app: kube-dns
      toPorts:
        - ports:
            - port: "53"
              protocol: UDP
            - port: "53"
              protocol: TCP

apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: api-server-policy
spec:
  endpointSelector:
    matchLabels:
      app: api-server
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
      authentication:
        mode: required
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
          rules:
            http:
              - method: GET
                path: "/api/v1/.*"
              - method: POST
                path: "/api/v1/orders"
  egress:
    - toEndpoints:
        - matchLabels:
            app: database
      toPorts:
        - ports:
            - port: "5432"
              protocol: TCP

# 查看 Endpoint 策略状态
$ cilium endpoint list
ENDPOINT   POLICY (ingress)   POLICY (egress)   IDENTITY   LABELS
2847       Enabled            Enabled           48372      k8s:app=frontend
3912       Enabled            Enabled           99001      k8s:app=api-server

# 模拟策略评估
$ cilium policy trace \
    --src-identity 48372 \
    --dst-identity 99001 \
    --dport 8080/TCP
Final verdict: ALLOWED

# 实时观察被拒绝的流量
$ hubble observe --verdict DROPPED
Oct 10 14:23:01: default/unknown-pod:54321 (ID:0)
  -> default/api-server-xyz:8080 (ID:99001)
  http-request DROPPED (HTTP/1.1 DELETE /api/v1/users)

# 按 Identity 过滤
$ hubble observe --from-identity 48372 --to-identity 99001
Oct 10 14:23:05: default/frontend-abc:38291 (ID:48372)
  -> default/api-server-xyz:8080 (ID:99001)
  http-request FORWARDED (HTTP/1.1 GET /api/v1/health)

# 创建 kind 集群
$ kind create cluster --name cilium-lab --config - <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
  disableDefaultCNI: true
  kubeProxyMode: none
nodes:
  - role: control-plane
  - role: worker
  - role: worker
EOF

# 安装 Cilium
$ helm install cilium cilium/cilium --version 1.16.4 \
    --namespace kube-system \
    --set kubeProxyReplacement=true \
    --set hubble.enabled=true \
    --set hubble.relay.enabled=true
$ cilium status --wait

# test-app.yaml -- api-server（nginx 模拟三个端点）
apiVersion: v1
kind: ConfigMap
metadata:
  name: api-nginx-conf
data:
  default.conf: |
    server {
        listen 80;
        location /api/v1/health { return 200 '{"status":"ok"}'; }
        location /api/v1/users  { return 200 '{"users":["alice","bob"]}'; }
        location /api/v1/admin  { return 200 '{"admin":true}'; }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-server
spec:
  replicas: 1
  selector: { matchLabels: { app: api-server } }
  template:
    metadata: { labels: { app: api-server, env: prod } }
    spec:
      containers:
        - name: api
          image: nginx:1.25
          ports: [{ containerPort: 80 }]
          volumeMounts: [{ name: conf, mountPath: /etc/nginx/conf.d }]
      volumes: [{ name: conf, configMap: { name: api-nginx-conf } }]
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: frontend
spec:
  replicas: 1
  selector: { matchLabels: { app: frontend } }
  template:
    metadata: { labels: { app: frontend, env: prod } }
    spec:
      containers:
        - { name: curl, image: "curlimages/curl:8.5.0", command: ["sleep","infinity"] }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: attacker
spec:
  replicas: 1
  selector: { matchLabels: { app: attacker } }
  template:
    metadata: { labels: { app: attacker } }
    spec:
      containers:
        - { name: curl, image: "curlimages/curl:8.5.0", command: ["sleep","infinity"] }

$ kubectl apply -f test-app.yaml
$ kubectl wait --for=condition=Ready pod -l app=api-server --timeout=60s
$ kubectl wait --for=condition=Ready pod -l app=frontend --timeout=60s

$ API_IP=$(kubectl get pod -l app=api-server -o jsonpath='{.items[0].status.podIP}')

# frontend 和 attacker 都能访问所有路径
$ kubectl exec deploy/frontend -- curl -s http://$API_IP/api/v1/admin
{"admin":true}
$ kubectl exec deploy/attacker -- curl -s http://$API_IP/api/v1/admin
{"admin":true}

# l7-policy.yaml
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: api-server-l7-policy
spec:
  endpointSelector:
    matchLabels:
      app: api-server
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
      toPorts:
        - ports:
            - port: "80"
              protocol: TCP
          rules:
            http:
              - method: GET
                path: "/api/v1/health"

$ kubectl apply -f l7-policy.yaml

# frontend -> GET /api/v1/health：允许（200）
$ kubectl exec deploy/frontend -- curl -s -o /dev/null -w "%{http_code}" \
    http://$API_IP/api/v1/health
200

# frontend -> GET /api/v1/users：L7 拒绝（403）
$ kubectl exec deploy/frontend -- curl -s -o /dev/null -w "%{http_code}" \
    http://$API_IP/api/v1/users
403

# frontend -> DELETE /api/v1/health：方法不匹配，L7 拒绝（403）
$ kubectl exec deploy/frontend -- curl -s -o /dev/null -w "%{http_code}" \
    -X DELETE http://$API_IP/api/v1/health
403

# attacker -> GET /api/v1/health：Identity 不匹配，L3/L4 直接丢弃
$ kubectl exec deploy/attacker -- curl -s --connect-timeout 5 \
    http://$API_IP/api/v1/health
# 超时 -- 流量在 eBPF 层面被丢弃，连接无法建立

$ hubble observe --namespace default --follow

# 允许的请求：
#   default/frontend-xxx (ID:48372) -> default/api-server-xxx:80 (ID:99001)
#   http-request FORWARDED (HTTP/1.1 GET /api/v1/health)

# L7 拒绝：
#   default/frontend-xxx (ID:48372) -> default/api-server-xxx:80 (ID:99001)
#   http-request DROPPED (HTTP/1.1 GET /api/v1/admin)

# L3/L4 拒绝：
#   default/attacker-xxx (ID:67890) -> default/api-server-xxx:80 (ID:99001)
#   Policy denied DROPPED (TCP Flags: SYN)

# 阶段 1：审计模式 -- 只记录不拦截
$ cilium config set policy-audit-mode enabled

# 用 Hubble 收集流量模式
$ hubble observe --verdict AUDIT --output json > traffic-baseline.json

# 阶段 2：分析基线，生成策略
# 阶段 3：测试环境验证
# 阶段 4：关闭审计模式，启用强制执行
$ cilium config set policy-audit-mode disabled

$ cilium identity list | wc -l   # 监控 Identity 数量

# Prometheus 告警规则
groups:
  - name: cilium-identity
    rules:
      - alert: CiliumIdentityAllocationFailure
        expr: rate(cilium_identity_allocation_errors_total[5m]) > 0
        for: 5m
        annotations:
          summary: "Cilium Identity 分配失败"
      - alert: CiliumPolicyDeniedSpike
        expr: rate(cilium_drop_count_total{reason="Policy denied"}[5m]) > 100
        for: 2m
        annotations:
          summary: "策略拒绝流量突然增加"
      - alert: CiliumL7ProxyLatencyHigh
        expr: histogram_quantile(0.99, rate(cilium_proxy_upstream_reply_seconds_bucket[5m])) > 0.5
        for: 5m
        annotations:
          summary: "L7 Proxy 延迟过高"