Loki Stack 生产实践：K8s 环境下日均 1T 日志的高可用架构

1
2
3

# Chunk 平均大小，健康值应接近 chunk_target_size（1.5MB）
# 持续低于 500KB 说明内存压力触发了提前 flush
loki_ingester_chunk_size_bytes_sum / loki_ingester_chunk_size_bytes_count

1
2
3
4
5
6
7

- pod_name    # Pod 每次重建名字不同，基数爆炸
- request_id  # UUID，基数 = 请求总数
- user_id     # 用户量级，基数不可控
- trace_id    # 放在日志正文，通过 LogQL 过滤
- ip_address  # IP 动态变化，基数高
- version     # 每次发版都变（app.kubernetes.io/version）
- pod_hash    # pod-template-hash，每个 RS 不同

1
2
3
4
5

# 高基数字段放在日志正文（JSON），通过 LogQL 精准过滤
{namespace="payment", app="order-service"}
  | json
  | trace_id="3fa2c1d4-8b7e-11ef-a3d2-0242ac120002"
  | line_format "{{.level}} {{.msg}}"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# object-storage-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: loki-s3-credentials
  namespace: logging
type: Opaque
stringData:
  AWS_ACCESS_KEY_ID: "YOUR_ACCESS_KEY"
  AWS_SECRET_ACCESS_KEY: "YOUR_SECRET_KEY"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# ingester-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: loki-wal-ssd
provisioner: kubernetes.io/aws-ebs
parameters:
  type: gp3
  iops: "3000"
  throughput: "125"
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

# loki-values.yaml
# helm install loki grafana/loki -n logging -f loki-values.yaml

loki:
  # ── 认证 ────────────────────────────────────────────────
  auth_enabled: true             # 生产环境开启多租户

  # ── 服务器 ──────────────────────────────────────────────
  server:
    http_listen_port: 3100
    grpc_listen_port: 9095
    log_level: info
    grpc_server_max_recv_msg_size: 104857600  # 100MB
    grpc_server_max_send_msg_size: 104857600

  # ── Memberlist（Gossip Ring） ────────────────────────────
  memberlist:
    join_members:
      - loki-memberlist.logging.svc.cluster.local:7946
    gossip_interval: 2s
    gossip_nodes: 3
    retransmit_factor: 4

  # ── 通用配置 ─────────────────────────────────────────────
  common:
    path_prefix: /loki
    replication_factor: 3        # 每条流写入 3 个 Ingester
    ring:
      kvstore:
        store: memberlist

  # ── 摄入限制 ─────────────────────────────────────────────
  limits_config:
    ingestion_rate_mb: 64        # 每租户 64MB/s 写入速率
    ingestion_burst_size_mb: 128
    per_stream_rate_limit: 10MB
    per_stream_rate_limit_burst: 20MB
    max_label_names_per_series: 15    # 标签数量上限，防止基数爆炸
    max_label_name_length: 1024
    max_label_value_length: 2048
    max_streams_per_user: 100000      # 每租户最多 10 万流
    max_query_series: 5000
    max_query_parallelism: 32
    max_entries_limit_per_query: 50000
    query_timeout: 5m
    retention_period: 30d
    unordered_writes: true            # 容忍网络抖动导致的乱序写入
    max_chunk_age: 2h

  # ── Ingester 配置（核心） ────────────────────────────────
  ingester:
    wal:
      enabled: true
      dir: /loki/wal
      flush_on_shutdown: true    # 下线前强制 flush，数据不丢
      replay_memory_ceiling: 6GB # WAL 回放内存上限
    lifecycler:
      ring:
        replication_factor: 3
        heartbeat_timeout: 1m
      heartbeat_period: 5s
      join_after: 30s            # 等待 Ring 稳定后再接受写入
      final_sleep: 30s           # 下线前等待 Ring 收敛
      num_tokens: 256            # 环上虚拟节点数，越多分布越均匀
    chunk_idle_period: 30m       # 流空闲超过 30 分钟触发 flush
    chunk_retain_period: 1m
    max_chunk_age: 2h            # 超过 2 小时强制 flush
    chunk_target_size: 1572864   # 目标 Chunk 大小 1.5MB
    chunk_encoding: snappy       # 压缩算法
    concurrent_flushes: 32       # 并发 flush goroutine 数
    max_transfer_retries: 0      # 使用 WAL 替代节点间 transfer

  # ── Distributor 配置 ─────────────────────────────────────
  distributor:
    ring:
      kvstore:
        store: memberlist

  # ── Compactor 配置 ───────────────────────────────────────
  compactor:
    working_directory: /loki/compactor
    shared_store: s3
    compaction_interval: 10m
    retention_enabled: true
    retention_delete_delay: 2h
    retention_delete_worker_count: 150

  # ── 存储配置（S3）────────────────────────────────────────
  storage_config:
    aws:
      s3: s3://ap-northeast-1/loki-prod-chunks
      region: ap-northeast-1
      access_key_id: ${AWS_ACCESS_KEY_ID}
      secret_access_key: ${AWS_SECRET_ACCESS_KEY}
      s3forcepathstyle: false
      http_config:
        idle_conn_timeout: 90s
    tsdb_shipper:
      active_index_directory: /loki/tsdb-index
      cache_location: /loki/tsdb-index-cache
      cache_ttl: 24h
      shared_store: s3

  # ── Schema 配置 ──────────────────────────────────────────
  schema_config:
    configs:
      - from: "2024-01-01"
        store: tsdb              # Loki 2.8+ 推荐，性能更优
        object_store: s3
        schema: v12
        index:
          prefix: loki_prod_index_
          period: 24h

  # ── Query 缓存 ───────────────────────────────────────────
  query_range:
    align_queries_with_step: true
    max_retries: 5
    cache_results: true
    results_cache:
      cache:
        redis:
          endpoint: redis:6379
          expiration: 1h

  querier:
    max_concurrent: 20
    query_ingesters_within: 3h   # 3h 内数据优先查 Ingester 热数据

  chunk_store_config:
    chunk_cache_config:
      redis:
        endpoint: redis:6379
        expiration: 2h

  frontend:
    log_queries_longer_than: 5s
    compress_responses: true
    max_outstanding_per_tenant: 2048

# ── K8s 组件资源配置 ─────────────────────────────────────────

# Ingester：有状态 StatefulSet，6 副本
ingester:
  replicas: 6
  maxUnavailable: 1
  persistence:
    enabled: true
    storageClass: loki-wal-ssd
    size: 50Gi                   # WAL 存储，按峰值 2h 写入量估算
  resources:
    requests:
      cpu: 2
      memory: 16Gi
    limits:
      cpu: 4
      memory: 24Gi
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              app.kubernetes.io/component: ingester
          topologyKey: kubernetes.io/hostname  # 禁止同节点部署
  tolerations:
    - key: "dedicated"
      operator: "Equal"
      value: "loki"
      effect: "NoSchedule"

# Distributor：无状态，自动扩缩
distributor:
  replicas: 3
  resources:
    requests:
      cpu: 1
      memory: 2Gi
    limits:
      cpu: 2
      memory: 4Gi
  autoscaling:
    enabled: true
    minReplicas: 3
    maxReplicas: 10
    targetCPUUtilizationPercentage: 60

# Query Frontend
queryFrontend:
  replicas: 2
  resources:
    requests:
      cpu: 1
      memory: 2Gi
    limits:
      cpu: 2
      memory: 4Gi

# Querier
querier:
  replicas: 3
  resources:
    requests:
      cpu: 2
      memory: 8Gi
    limits:
      cpu: 4
      memory: 16Gi
  autoscaling:
    enabled: true
    minReplicas: 3
    maxReplicas: 12
    targetCPUUtilizationPercentage: 70

# Compactor：单实例（Leader 选举保证唯一）
compactor:
  replicas: 1
  resources:
    requests:
      cpu: 500m
      memory: 2Gi
    limits:
      cpu: 2
      memory: 4Gi

# Ruler
ruler:
  enabled: true
  replicas: 2
  resources:
    requests:
      cpu: 500m
      memory: 1Gi
    limits:
      cpu: 1
      memory: 2Gi

# Redis 缓存（Query 结果 + Chunk 缓存）
redis:
  enabled: true
  auth:
    enabled: true
    password: "YOUR_REDIS_PASSWORD"
  master:
    resources:
      requests:
        cpu: 500m
        memory: 4Gi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

# fluent-bit-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: logging
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush           5
        Daemon          Off
        Log_Level       info
        Parsers_File    parsers.conf
        HTTP_Server     On
        HTTP_Listen     0.0.0.0
        HTTP_Port       2020        # 暴露 /metrics 给 Prometheus 抓取

    # ── INPUT：读取所有 Pod 日志 ─────────────────────────────
    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        Exclude_Path      /var/log/containers/*_logging_*.log  # 排除自身
        Parser            cri
        DB                /var/log/flb_kube.db  # 记录读取位置，重启不丢失
        DB.Sync           Normal
        Mem_Buf_Limit     50MB
        Skip_Long_Lines   On
        Refresh_Interval  10

    # ── FILTER 1：注入 K8s metadata ─────────────────────────
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        Merge_Log           On      # 将 JSON 正文字段合并到顶层
        Merge_Log_Key       log_processed
        Keep_Log            Off     # 合并后丢弃原始 log 字段
        Annotations         Off     # 不注入 annotation（避免基数爆炸）
        Labels              On      # 注入 Pod labels

    # ── FILTER 2：提升 JSON 日志字段到顶层 ──────────────────
    [FILTER]
        Name          nest
        Match         kube.*
        Operation     lift
        Nested_under  log_processed
        Add_prefix    parsed_

    # ── FILTER 3：丢弃 debug 日志（生产降噪）────────────────
    [FILTER]
        Name    grep
        Match   kube.*
        Exclude parsed_level ^debug$

    # ── OUTPUT：推送到 Loki ──────────────────────────────────
    [OUTPUT]
        Name            loki
        Match           kube.*
        Host            loki-distributor.logging.svc.cluster.local
        Port            3100
        Labels          job=fluentbit, cluster=prod-cn-north
        # 核心：只映射低基数字段为 Loki 标签
        Label_Keys      $kubernetes['namespace_name'],$kubernetes['labels']['app.kubernetes.io/name'],$parsed_level
        Tenant_ID       production
        Line_Format     json
        # ⚠️ 关键：禁止自动映射所有 k8s label，否则基数爆炸
        Auto_Kubernetes_Labels Off
        Retry_Limit     5

  parsers.conf: |
    # containerd / CRI 日志格式
    [PARSER]
        Name        cri
        Format      regex
        Regex       ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z

    # 应用结构化 JSON 日志
    [PARSER]
        Name        json
        Format      json
        Time_Key    timestamp
        Time_Format %Y-%m-%dT%H:%M:%S.%LZ

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

# fluent-bit-daemonset.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluent-bit
  namespace: logging
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluent-bit
rules:
  - apiGroups: [""]
    resources: ["pods", "namespaces", "nodes"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluent-bit
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: fluent-bit
subjects:
  - kind: ServiceAccount
    name: fluent-bit
    namespace: logging
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: logging
spec:
  selector:
    matchLabels:
      app: fluent-bit
  template:
    metadata:
      labels:
        app: fluent-bit
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "2020"
        prometheus.io/path: "/api/v1/metrics/prometheus"
    spec:
      serviceAccountName: fluent-bit
      tolerations:
        - operator: Exists             # 采集所有节点，含 master/taint 节点
      containers:
        - name: fluent-bit
          image: fluent/fluent-bit:3.2
          resources:
            requests:
              cpu: 100m
              memory: 64Mi            # 对比 Promtail 的 256Mi，节省 4 倍
            limits:
              cpu: 500m
              memory: 128Mi
          ports:
            - containerPort: 2020
              name: metrics
          livenessProbe:
            httpGet:
              path: /
              port: 2020
            initialDelaySeconds: 10
            periodSeconds: 30
          volumeMounts:
            - name: config
              mountPath: /fluent-bit/etc/
            - name: varlog
              mountPath: /var/log
              readOnly: true
            - name: db
              mountPath: /var/log/flb_kube.db
      volumes:
        - name: config
          configMap:
            name: fluent-bit-config
        - name: varlog
          hostPath:
            path: /var/log
        - name: db
          hostPath:
            path: /var/log/flb_kube.db

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# ingester-scaledobject.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: loki-ingester-scaler
  namespace: logging
spec:
  scaleTargetRef:
    name: loki-ingester
  minReplicaCount: 6
  maxReplicaCount: 16
  triggers:
    - type: prometheus
      metadata:
        serverAddress: http://prometheus:9090
        metricName: loki_ingester_chunks_per_second
        threshold: "500"          # 每实例超过 500 chunks/s 触发扩容
        query: |
          sum(rate(loki_ingester_chunks_created_total[2m])) /
          count(up{job="loki-ingester"})

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

# loki-rules-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-alert-rules
  namespace: logging
data:
  production-rules.yaml: |
    groups:
      - name: production.critical
        interval: 1m
        rules:
          # 错误日志速率过高
          - alert: HighErrorLogRate
            expr: |
              sum(rate({namespace=~"payment|order|user", level="error"}[5m])) by (namespace, app)
              > 10
            for: 2m
            labels:
              severity: warning
              team: backend
            annotations:
              summary: "{{ $labels.app }} 错误日志速率过高"
              description: |
                应用 {{ $labels.app }}（namespace: {{ $labels.namespace }}）
                过去 5 分钟错误日志速率 {{ $value | humanize }} 条/秒，超过阈值 10 条/秒。

          # OOM 事件检测
          - alert: PodOOMKilled
            expr: |
              count_over_time({namespace=~".+"} |= "OOMKilled" [5m]) > 0
            labels:
              severity: critical
            annotations:
              summary: "检测到 OOMKilled 事件"

          # 日志写入速率骤降（采集断路预警）
          - alert: LokiIngestionDropped
            expr: |
              sum(rate(loki_distributor_lines_received_total[5m]))
              < sum(rate(loki_distributor_lines_received_total[5m] offset 30m)) * 0.5
            for: 5m
            labels:
              severity: critical
              team: platform
            annotations:
              summary: "Loki 日志写入速率骤降超过 50%"

      - name: loki.infrastructure
        interval: 2m
        rules:
          # Ingester 流数量预警（标签基数健康度）
          - alert: LokiHighStreamCount
            expr: |
              sum(loki_ingester_streams_created_total) by (pod) > 50000
            for: 5m
            labels:
              severity: warning
            annotations:
              summary: "{{ $labels.pod }} 流数量过高，检查标签基数"

          # Chunk 平均大小过小（内存压力导致提前 flush）
          - alert: LokiSmallChunkSize
            expr: |
              (loki_ingester_chunk_size_bytes_sum / loki_ingester_chunk_size_bytes_count) < 524288
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "Ingester Chunk 平均大小 < 512KB，可能存在内存压力"

          # Ingester 内存 Chunks 积压
          - alert: LokiFlushQueueDepth
            expr: loki_ingester_memory_chunks > 50000
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "Ingester 内存 Chunks 积压，Flush 速率可能不足"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# grafana-datasource.yaml
apiVersion: 1
datasources:
  - name: Loki-Production
    type: loki
    access: proxy
    url: http://loki-query-frontend.logging.svc.cluster.local:3100
    jsonData:
      maxLines: 5000
      timeout: 300
      httpHeaderName1: "X-Scope-OrgID"
    secureJsonData:
      httpHeaderValue1: "production"
    isDefault: false
    editable: true

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

# 0. 准备 namespace 和前置资源
kubectl create namespace logging
kubectl apply -f object-storage-secret.yaml
kubectl apply -f ingester-storageclass.yaml

# 1. 添加 Helm 仓库
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

# 2. 安装 Loki（分布式模式）
helm install loki grafana/loki \
  --namespace logging \
  --version 6.x.x \
  -f loki-values.yaml \
  --set loki.storage.s3.accessKeyId="${AWS_ACCESS_KEY_ID}" \
  --set loki.storage.s3.secretAccessKey="${AWS_SECRET_ACCESS_KEY}"

# 3. 部署 Fluent Bit
kubectl apply -f fluent-bit-configmap.yaml
kubectl apply -f fluent-bit-daemonset.yaml

# 4. 部署 KEDA 弹性扩容（需提前安装 KEDA）
kubectl apply -f ingester-scaledobject.yaml

# 5. 应用告警规则
kubectl apply -f loki-rules-configmap.yaml

# 6. 验证
kubectl get pods -n logging
# 检查 Ring 状态（所有 Ingester 应为 ACTIVE）
kubectl exec -n logging deploy/loki-distributor -- \
  curl -s http://localhost:3100/ring | python3 -m json.tool
# 检查 Fluent Bit 写入成功率
kubectl exec -n logging ds/fluent-bit -- \
  curl -s http://localhost:2020/api/v1/metrics/prometheus | grep fluentbit_output_retried_records

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 查看 Ring 状态（所有节点应为 ACTIVE）
curl -s http://loki-ingester:3100/ring | python3 -m json.tool

# 查看关键指标
curl -s http://loki-ingester:3100/metrics | grep -E \
  "loki_ingester_(streams|chunks|memory|flush|chunk_size)"

# 手动触发 Flush（维护前执行）
curl -X POST http://loki-ingester-0:3100/flush

# 优雅下线（缩容前执行，等待 flush 完成后再删 Pod）
curl -X POST http://loki-ingester-5:3100/ingester/shutdown