# 格式: <module>_<name>_<unit>

# 正确示例
http_requests_total              # HTTP 请求总数
http_request_duration_seconds    # HTTP 请求延迟 (带单位)
node_memory_usage_bytes         # 节点内存使用 (带单位)
process_cpu_seconds_total        # 进程 CPU 时间

# 错误示例
http_request_count               # 缺少单位
HTTPRequests                     # 使用驼峰命名
requests_per_second              # 缺少前缀

# 推荐标签
{method="GET", status="200", handler="/api/users"}

# 不推荐
{METHOD="GET", STATUS="200"}  # 全大写
{UserId="123"}                # 驼峰命名

# ❌ 高基数 - 每个请求都有 unique_id
{__name__="http_request", unique_id="req_12345"}

# ✅ 低基数 - 按状态码分组
{__name__="http_request", status="200"}

# ❌ 高基数 - 用户 ID
{__name__="api_call", user_id="12345"}

# ✅ 低基数 - 按 API 端点
{__name__="api_call", endpoint="/api/users"}

# ✅ 好的标签设计
{method="GET", status="200", handler="/api/users", env="prod"}

# ✅ 使用 relabel 减少标签
# prometheus.yml
relabel_configs:
  - source_labels: [__meta_kubernetes_pod_label_app]
    regex: '(.+)'
    replacement: '${1}'
    target_label: app

# ✅ 使用 drop 过滤无效数据
relabel_configs:
  - source_labels: [instance]
    regex: 'test-.*'
    action: drop

scrape_configs:
  - job_name: 'optimized-scrape'
    # 合理的抓取间隔
    scrape_interval: 30s
    
    # 抓取超时 (应小于间隔)
    scrape_timeout: 20s
    
    # 限制标签长度
    label_limit: 64
    
    # 限制标签名长度
    label_name_length_limit: 512
    
    # 限制标签值长度
    label_value_length_limit: 2048

# ❌ 低效: 全扫描
sum(http_requests_total)

# ✅ 高效: 使用标签过滤
sum(http_requests_total{job="api", env="prod"})

# ❌ 低效: 多个全扫描
sum(rate(http_requests_total[5m])) / sum(rate(http_requests_total[5m]))

# ✅ 高效: 子查询限制范围
avg_over_time(http_requests_total{job="api"}[1h:1m])

# ❌ 低效: 复杂正则
{job=~"api.*|web.*|admin.*"}

# ✅ 高效: 简单匹配
{job=~"api|web|admin"}

groups:
  - name: query_optimization
    interval: 30s
    rules:
      # 预计算高频查询
      - record: job:api_requests:rate5m
        expr: sum by (job) (rate(http_requests_total[5m]))
      
      # 预计算告警阈值
      - record: alert:cpu_threshold_exceeded
        expr: avg by (instance) (rate(node_cpu_seconds_total[5m])) > 0.8

storage:
  tsdb:
    # 合理的数据保留期
    retention:
      time: 30d
      size: 100GB
    
    # 启用压缩
    wal_compression: true
    
    # 控制块大小
    block_duration: 2h

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

# 检查 Prometheus 健康状态
- alert: PrometheusNotHealthy
  expr: prometheus_up != 1
  for: 5m
  labels:
    severity: critical

# 监控 Exporter 存活
up{job="node-exporter"}

# 监控 Exporter 指标数量
count by (job) ({__name__=~".+"})

# 监控指标新鲜度
time() - max by (job) (timestamp(up{job="node-exporter"}))

groups:
  - name: monitoring_health
    rules:
      # Exporter 不可用
      - alert: ExporterDown
        expr: up == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.job }} 不可用"
          
      # 指标陈旧
      - alert: MetricsStale
        expr: |
          (time() - timestamp(up)) > 300
        for: 5m
        labels:
          severity: warning

# 延迟 (Latency)
histogram_quantile(0.99, 
  sum by (le, service) (rate(http_request_duration_seconds_bucket[5m]))
)

# 流量 (Traffic)
sum by (service) (rate(http_requests_total[5m]))

# 错误 (Errors)
sum by (service) (rate(http_requests_total{status=~"5.."}[5m])) 
  / 
sum by (service) (rate(http_requests_total[5m]))

# 饱和度 (Saturation)
instance:node_cpu_usage:percent > 80

# 备份 TSDB
tar -czf prometheus_backup_$(date +%Y%m%d).tar.gz /data/prometheus

# 备份配置
cp /etc/prometheus/prometheus.yml /backup/prometheus.yml.$(date +%Y%m%d)

# 备份规则
tar -czf prometheus_rules_$(date +%Y%m%d).tar.gz /etc/prometheus/rules

# 1. 备份
cp -r /data/prometheus /data/prometheus.bak

# 2. 下载新版本
PROMETHEUS_VERSION="2.48.0"
wget https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz

# 3. 停止服务
sudo systemctl stop prometheus

# 4. 替换二进制
tar xzf prometheus-${PROMETHEUS_VERSION}.linux-amd64.tar.gz
sudo rm -rf /usr/local/prometheus_old
sudo mv /usr/local/prometheus /usr/local/prometheus_old
sudo mv prometheus-${PROMETHEUS_VERSION}.linux-amd64 /usr/local/prometheus

# 5. 启动服务
sudo systemctl start prometheus

# 6. 验证
curl http://localhost:9090/-/healthy

# 容量估算公式

# 每日样本数
daily_samples = targets * (60 / scrape_interval) * 60 * 24 * metrics_per_target

# 存储估算 (GB/天)
daily_storage_gb = daily_samples * sample_size_bytes / (1024**3) / compression_ratio

# 内存估算
memory_gb = (targets * avg_series_per_target * avg_labels * label_size) / (1024**3)

# 示例计算
# 100 targets, 15s interval, 500 metrics/target, 50 samples/day
targets = 100
interval = 15
metrics = 500
sample_size = 2  # bytes (压缩后)
compression = 10

daily_samples = 100 * (60/15) * 60 * 24 * 500  # 288,000,000
daily_gb = daily_samples * 2 / compression / (1024**3)  # ~0.5 GB/day
30day_gb = 30 * daily_gb  # ~15 GB

# http_requests_total

HTTP 请求计数器，记录所有 HTTP 请求。

## 标签

| 标签 | 类型 | 说明 |
|------|------|------|
| method | string | HTTP 方法 (GET, POST, PUT, DELETE) |
| status | string | HTTP 状态码 (200, 404, 500) |
| handler | string | 请求处理器路径 |
| job | string | 目标服务名称 |

## 示例

```promql
# 按 handler 统计 QPS
sum by (handler) (rate(http_requests_total[5m]))

# 错误率
sum(rate(http_requests_total{status=~"5.."}[5m])) 
  / 
sum(rate(http_requests_total[5m]))

### 13.7.2 SLO 定义

```yaml
# SLO 配置
groups:
  - name: slo_availability
    interval: 1m
    rules:
      # 请求成功率
      - record: slo:request_success:ratio5m
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[5m]))
            /
          sum(rate(http_requests_total[5m]))
      
      # 30 天窗口
      - record: slo:request_success:ratio30d
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[30d]))
            /
          sum(rate(http_requests_total[30d]))
      
      # 错误预算
      - record: slo:error_budget_remaining:ratio
        expr: |
          (1 - (1 - slo:request_success:ratio30d) / 0.99) * 100